KEYCLOAK-4189 - Cross DC testing

common/src/main/java/org/keycloak/common/util/SystemEnvProperties.java

@@ -17,6 +17,8 @@
 
 package org.keycloak.common.util;
 
+import java.util.Collections;
+import java.util.Map;
 import java.util.Properties;
 
 /**
@@ -24,9 +26,21 @@ import java.util.Properties;
  */
 public class SystemEnvProperties extends Properties {
 
+    private final Map<String, String> overrides;
+
+    public SystemEnvProperties(Map<String, String> overrides) {
+        this.overrides = overrides;
+    }
+
+    public SystemEnvProperties() {
+        this.overrides = Collections.EMPTY_MAP;
+    }
+
     @Override
     public String getProperty(String key) {
-        if (key.startsWith("env.")) {
+        if (overrides.containsKey(key)) {
+            return overrides.get(key);
+        } else if (key.startsWith("env.")) {
             return System.getenv().get(key.substring(4));
         } else {
             return System.getProperty(key);

misc/CrossDataCenter.md

@@ -3,6 +3,8 @@ Test Cross-Data-Center scenario (test with external JDG server)
 
 These are temporary notes. This docs should be removed once we have cross-DC support finished and properly documented. 
 
+Note that these steps are already automated, see Cross-DC tests section in [HOW-TO-RUN.md](../testsuite/integration-arquillian/HOW-TO-RUN.md) document.
+ 
 What is working right now is:
 - Propagating of invalidation messages for "realms" and "users" caches
 - All the other things provided by ClusterProvider, which is:
@@ -18,7 +20,7 @@ Basic setup
 
 This is setup with 2 keycloak nodes, which are NOT in cluster. They just share the same database and they will be configured with "work" infinispan cache with remoteStore, which will point
 to external JDG server.
- 
+
 JDG Server setup
 ----------------
 - Download JDG 7.0 server and unzip to some folder

model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java

@@ -154,7 +154,8 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
 
         if (clustered) {
             String nodeName = config.get("nodeName", System.getProperty(InfinispanConnectionProvider.JBOSS_NODE_NAME));
-            configureTransport(gcb, nodeName);
+            String jgroupsUdpMcastAddr = config.get("jgroupsUdpMcastAddr", System.getProperty(InfinispanConnectionProvider.JGROUPS_UDP_MCAST_ADDR));
+            configureTransport(gcb, nodeName, jgroupsUdpMcastAddr);
         }
         gcb.globalJmxStatistics().allowDuplicateDomains(allowDuplicateJMXDomains);
 
@@ -317,24 +318,40 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
         return cb.build();
     }
 
-    protected void configureTransport(GlobalConfigurationBuilder gcb, String nodeName) {
+    private static final Object CHANNEL_INIT_SYNCHRONIZER = new Object();
+
+    protected void configureTransport(GlobalConfigurationBuilder gcb, String nodeName, String jgroupsUdpMcastAddr) {
         if (nodeName == null) {
             gcb.transport().defaultTransport();
         } else {
             FileLookup fileLookup = FileLookupFactory.newInstance();
 
-            try {
+            synchronized (CHANNEL_INIT_SYNCHRONIZER) {
-                // Compatibility with Wildfly
+                String originalMcastAddr = System.getProperty(InfinispanConnectionProvider.JGROUPS_UDP_MCAST_ADDR);
-                JChannel channel = new JChannel(fileLookup.lookupFileLocation("default-configs/default-jgroups-udp.xml", this.getClass().getClassLoader()));
+                if (jgroupsUdpMcastAddr == null) {
-                channel.setName(nodeName);
+                    System.getProperties().remove(InfinispanConnectionProvider.JGROUPS_UDP_MCAST_ADDR);
-                JGroupsTransport transport = new JGroupsTransport(channel);
+                } else {
+                    System.setProperty(InfinispanConnectionProvider.JGROUPS_UDP_MCAST_ADDR, jgroupsUdpMcastAddr);
+                }
+                try {
+                    // Compatibility with Wildfly
+                    JChannel channel = new JChannel(fileLookup.lookupFileLocation("default-configs/default-jgroups-udp.xml", this.getClass().getClassLoader()));
+                    channel.setName(nodeName);
+                    JGroupsTransport transport = new JGroupsTransport(channel);
 
-                gcb.transport().nodeName(nodeName);
+                    gcb.transport().nodeName(nodeName);
-                gcb.transport().transport(transport);
+                    gcb.transport().transport(transport);
 
-                logger.infof("Configured jgroups transport with the channel name: %s", nodeName);
+                    logger.infof("Configured jgroups transport with the channel name: %s", nodeName);
-            } catch (Exception e) {
+                } catch (Exception e) {
-                throw new RuntimeException(e);
+                    throw new RuntimeException(e);
+                } finally {
+                    if (originalMcastAddr == null) {
+                        System.getProperties().remove(InfinispanConnectionProvider.JGROUPS_UDP_MCAST_ADDR);
+                    } else {
+                        System.setProperty(InfinispanConnectionProvider.JGROUPS_UDP_MCAST_ADDR, originalMcastAddr);
+                    }
+                }
             }
         }
     }

model/infinispan/src/main/java/org/keycloak/connections/infinispan/InfinispanConnectionProvider.java

@@ -53,6 +53,7 @@ public interface InfinispanConnectionProvider extends Provider {
 
     // System property used on Wildfly to identify distributedCache address and sticky session route
     String JBOSS_NODE_NAME = "jboss.node.name";
+    String JGROUPS_UDP_MCAST_ADDR = "jgroups.udp.mcast_addr";
 
 
     <K, V> Cache<K, V> getCache(String name);

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/CacheManager.java

@@ -220,7 +220,7 @@ public abstract class CacheManager {
 
         addInvalidationsFromEvent(event, invalidations);
 
-        getLogger().debugf("Invalidating %d cache items after received event %s", invalidations.size(), event);
+        getLogger().debugf("[%s] Invalidating %d cache items after received event %s", cache.getCacheManager().getAddress(), invalidations.size(), event);
 
         for (String invalidation : invalidations) {
             invalidateObject(invalidation);

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/events/AddInvalidatedActionTokenEvent.java

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.keycloak.models.cache.infinispan;
+package org.keycloak.models.cache.infinispan.events;
 
 import org.keycloak.cluster.ClusterEvent;
 import org.keycloak.models.sessions.infinispan.entities.ActionTokenReducedKey;

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/events/RemoveActionTokensSpecificEvent.java

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.keycloak.models.cache.infinispan;
+package org.keycloak.models.cache.infinispan.events;
 
 import org.keycloak.cluster.ClusterEvent;
 

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanActionTokenStoreProvider.java

@@ -19,8 +19,8 @@ package org.keycloak.models.sessions.infinispan;
 import org.keycloak.cluster.ClusterProvider;
 import org.keycloak.models.*;
 
-import org.keycloak.models.cache.infinispan.AddInvalidatedActionTokenEvent;
+import org.keycloak.models.cache.infinispan.events.AddInvalidatedActionTokenEvent;
-import org.keycloak.models.cache.infinispan.RemoveActionTokensSpecificEvent;
+import org.keycloak.models.cache.infinispan.events.RemoveActionTokensSpecificEvent;
 import org.keycloak.models.sessions.infinispan.entities.ActionTokenValueEntity;
 import org.keycloak.models.sessions.infinispan.entities.ActionTokenReducedKey;
 import java.util.*;
@@ -58,7 +58,12 @@ public class InfinispanActionTokenStoreProvider implements ActionTokenStoreProvi
         ActionTokenValueEntity tokenValue = new ActionTokenValueEntity(notes);
 
         ClusterProvider cluster = session.getProvider(ClusterProvider.class);
-        this.tx.notify(cluster, InfinispanActionTokenStoreProviderFactory.ACTION_TOKEN_EVENTS, new AddInvalidatedActionTokenEvent(tokenKey, key.getExpiration(), tokenValue), false);
+        AddInvalidatedActionTokenEvent event = new AddInvalidatedActionTokenEvent(tokenKey, key.getExpiration(), tokenValue);
+        this.tx.notify(cluster, generateActionTokenEventId(), event, false);
+    }
+
+    private static String generateActionTokenEventId() {
+        return InfinispanActionTokenStoreProviderFactory.ACTION_TOKEN_EVENTS + "/" + UUID.randomUUID();
     }
 
     @Override
@@ -93,6 +98,6 @@ public class InfinispanActionTokenStoreProvider implements ActionTokenStoreProvi
         }
 
         ClusterProvider cluster = session.getProvider(ClusterProvider.class);
-        this.tx.notify(cluster, InfinispanActionTokenStoreProviderFactory.ACTION_TOKEN_EVENTS, new RemoveActionTokensSpecificEvent(userId, actionId), false);
+        this.tx.notify(cluster, generateActionTokenEventId(), new RemoveActionTokensSpecificEvent(userId, actionId), false);
     }
 }

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanActionTokenStoreProviderFactory.java

@@ -23,14 +23,16 @@ import org.keycloak.common.util.Time;
 import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
 import org.keycloak.models.*;
 
-import org.keycloak.models.cache.infinispan.AddInvalidatedActionTokenEvent;
+import org.keycloak.models.cache.infinispan.events.AddInvalidatedActionTokenEvent;
-import org.keycloak.models.cache.infinispan.RemoveActionTokensSpecificEvent;
+import org.keycloak.models.cache.infinispan.events.RemoveActionTokensSpecificEvent;
 import org.keycloak.models.sessions.infinispan.entities.ActionTokenValueEntity;
 import org.keycloak.models.sessions.infinispan.entities.ActionTokenReducedKey;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 import org.infinispan.Cache;
 import org.infinispan.context.Flag;
+import org.infinispan.remoting.transport.Address;
+import org.jboss.logging.Logger;
 
 /**
  *
@@ -38,6 +40,10 @@ import org.infinispan.context.Flag;
  */
 public class InfinispanActionTokenStoreProviderFactory implements ActionTokenStoreProviderFactory {
 
+    private static final Logger LOG = Logger.getLogger(InfinispanActionTokenStoreProviderFactory.class);
+
+    private volatile Cache<ActionTokenReducedKey, ActionTokenValueEntity> actionTokenCache;
+
     public static final String ACTION_TOKEN_EVENTS = "ACTION_TOKEN_EVENTS";
 
     /**
@@ -49,34 +55,7 @@ public class InfinispanActionTokenStoreProviderFactory implements ActionTokenSto
 
     @Override
     public ActionTokenStoreProvider create(KeycloakSession session) {
-        InfinispanConnectionProvider connections = session.getProvider(InfinispanConnectionProvider.class);
+        return new InfinispanActionTokenStoreProvider(session, this.actionTokenCache);
-        Cache<ActionTokenReducedKey, ActionTokenValueEntity> actionTokenCache = connections.getCache(InfinispanConnectionProvider.ACTION_TOKEN_CACHE);
-
-        ClusterProvider cluster = session.getProvider(ClusterProvider.class);
-
-        cluster.registerListener(ACTION_TOKEN_EVENTS, event -> {
-            if (event instanceof RemoveActionTokensSpecificEvent) {
-                RemoveActionTokensSpecificEvent e = (RemoveActionTokensSpecificEvent) event;
-
-                actionTokenCache
-                  .getAdvancedCache()
-                  .withFlags(Flag.CACHE_MODE_LOCAL, Flag.SKIP_CACHE_LOAD)
-                  .keySet()
-                  .stream()
-                  .filter(k -> Objects.equals(k.getUserId(), e.getUserId()) && Objects.equals(k.getActionId(), e.getActionId()))
-                  .forEach(actionTokenCache::remove);
-            } else if (event instanceof AddInvalidatedActionTokenEvent) {
-                AddInvalidatedActionTokenEvent e = (AddInvalidatedActionTokenEvent) event;
-
-                if (e.getExpirationInSecs() == DEFAULT_CACHE_EXPIRATION) {
-                    actionTokenCache.put(e.getKey(), e.getTokenValue());
-                } else {
-                    actionTokenCache.put(e.getKey(), e.getTokenValue(), e.getExpirationInSecs() - Time.currentTime(), TimeUnit.SECONDS);
-                }
-            }
-        });
-
-        return new InfinispanActionTokenStoreProvider(session, actionTokenCache);
     }
 
     @Override
@@ -84,8 +63,57 @@ public class InfinispanActionTokenStoreProviderFactory implements ActionTokenSto
         this.config = config;
     }
 
+    private static Cache<ActionTokenReducedKey, ActionTokenValueEntity> initActionTokenCache(KeycloakSession session) {
+        InfinispanConnectionProvider connections = session.getProvider(InfinispanConnectionProvider.class);
+        Cache<ActionTokenReducedKey, ActionTokenValueEntity> cache = connections.getCache(InfinispanConnectionProvider.ACTION_TOKEN_CACHE);
+        final Address cacheAddress = cache.getCacheManager().getAddress();
+
+        ClusterProvider cluster = session.getProvider(ClusterProvider.class);
+
+        cluster.registerListener(ClusterProvider.ALL, event -> {
+            if (event instanceof RemoveActionTokensSpecificEvent) {
+                RemoveActionTokensSpecificEvent e = (RemoveActionTokensSpecificEvent) event;
+
+                LOG.debugf("[%s] Removing token invalidation for user+action: userId=%s, actionId=%s", cacheAddress, e.getUserId(), e.getActionId());
+
+                cache
+                  .getAdvancedCache()
+                  .withFlags(Flag.CACHE_MODE_LOCAL, Flag.SKIP_CACHE_LOAD)
+                  .keySet()
+                  .stream()
+                  .filter(k -> Objects.equals(k.getUserId(), e.getUserId()) && Objects.equals(k.getActionId(), e.getActionId()))
+                  .forEach(cache::remove);
+            } else if (event instanceof AddInvalidatedActionTokenEvent) {
+                AddInvalidatedActionTokenEvent e = (AddInvalidatedActionTokenEvent) event;
+
+                LOG.debugf("[%s] Invalidating token %s", cacheAddress, e.getKey());
+                if (e.getExpirationInSecs() == DEFAULT_CACHE_EXPIRATION) {
+                    cache.put(e.getKey(), e.getTokenValue());
+                } else {
+                    cache.put(e.getKey(), e.getTokenValue(), e.getExpirationInSecs() - Time.currentTime(), TimeUnit.SECONDS);
+                }
+            }
+        });
+
+        LOG.debugf("[%s] Registered cluster listeners", cacheAddress);
+
+        return cache;
+    }
+
     @Override
     public void postInit(KeycloakSessionFactory factory) {
+        Cache<ActionTokenReducedKey, ActionTokenValueEntity> cache = this.actionTokenCache;
+
+        // It is necessary to put the cache initialization here, otherwise the cache would be initialized lazily, that
+        // means also listeners will start only after first cache initialization - that would be too late
+        if (cache == null) {
+            synchronized (this) {
+                cache = this.actionTokenCache;
+                if (cache == null) {
+                    this.actionTokenCache = initActionTokenCache(factory.create());
+                }
+            }
+        }
     }
 
     @Override

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanAuthenticationSessionProviderFactory.java

@@ -92,7 +92,7 @@ public class InfinispanAuthenticationSessionProviderFactory implements Authentic
                     ClusterProvider cluster = session.getProvider(ClusterProvider.class);
                     cluster.registerListener(AUTHENTICATION_SESSION_EVENTS, this::updateAuthNotes);
 
-                    log.debug("Registered cluster listeners");
+                    log.debugf("[%s] Registered cluster listeners", authSessionsCache.getCacheManager().getAddress());
                 }
             }
         }

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/entities/ActionTokenReducedKey.java

@@ -81,6 +81,11 @@ public class ActionTokenReducedKey implements Serializable {
           && Objects.equals(this.actionVerificationNonce, other.getActionVerificationNonce());
     }
 
+    @Override
+    public String toString() {
+        return "userId=" + userId + ", actionId=" + actionId + ", actionVerificationNonce=" + actionVerificationNonce;
+    }
+
     public static class ExternalizerImpl implements Externalizer<ActionTokenReducedKey> {
 
         @Override

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/entities/ActionTokenValueEntity.java

@@ -53,7 +53,7 @@ public class ActionTokenValueEntity implements ActionTokenValueModel {
         public void writeObject(ObjectOutput output, ActionTokenValueEntity t) throws IOException {
             output.writeByte(VERSION_1);
 
-            output.writeBoolean(! t.notes.isEmpty());
+            output.writeBoolean(t.notes.isEmpty());
             if (! t.notes.isEmpty()) {
                 output.writeObject(t.notes);
             }

services/src/main/java/org/keycloak/services/resources/KeycloakApplication.java

@@ -70,8 +70,10 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.net.URI;
 import java.net.URL;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
 import java.util.StringTokenizer;
@@ -88,6 +90,8 @@ public class KeycloakApplication extends Application {
 
     public static final String KEYCLOAK_EMBEDDED = "keycloak.embedded";
 
+    public static final String SERVER_CONTEXT_CONFIG_PROPERTY_OVERRIDES = "keycloak.server.context.config.property-overrides";
+
     private static final Logger logger = Logger.getLogger(KeycloakApplication.class);
 
     protected boolean embedded = false;
@@ -262,7 +266,7 @@ public class KeycloakApplication extends Application {
     public static void loadConfig(ServletContext context) {
         try {
             JsonNode node = null;
-            
+
             String dmrConfig = loadDmrConfig(context);
             if (dmrConfig != null) {
                 node = new ObjectMapper().readTree(dmrConfig);
@@ -287,7 +291,13 @@ public class KeycloakApplication extends Application {
             }
 
             if (node != null) {
-                Properties properties = new SystemEnvProperties();
+                Map<String, String> propertyOverridesMap = new HashMap<>();
+                String propertyOverrides = context.getInitParameter(SERVER_CONTEXT_CONFIG_PROPERTY_OVERRIDES);
+                if (context.getInitParameter(SERVER_CONTEXT_CONFIG_PROPERTY_OVERRIDES) != null) {
+                    JsonNode jsonObj = new ObjectMapper().readTree(propertyOverrides);
+                    jsonObj.fields().forEachRemaining(e -> propertyOverridesMap.put(e.getKey(), e.getValue().asText()));
+                }
+                Properties properties = new SystemEnvProperties(propertyOverridesMap);
                 Config.init(new JsonConfigProvider(node, properties));
             } else {
                 throw new RuntimeException("Keycloak config not found.");

testsuite/integration-arquillian/HOW-TO-RUN.md

@@ -417,4 +417,22 @@ and argument: `-p 8181`
 
 3) Run loadbalancer (class `SimpleUndertowLoadBalancer`) without arguments and system properties. Loadbalancer runs on port 8180, so you can access Keycloak on `http://localhost:8180/auth`     
 
+## Cross-DC tests
 
+Cross-DC tests use 2 data centers, each with one automatically started and one manually controlled backend servers
+(currently only Keycloak on Undertow), and 1 frontend loadbalancer server node that sits in front of all servers.
+The browser usually communicates directly with the frontent node and the test controls where the HTTP requests
+land by adjusting load balancer configuration (e.g. to direct the traffic to only a single DC).
+
+For an example of a test, see [org.keycloak.testsuite.crossdc.ActionTokenCrossDCTest](tests/base/src/test/java/org/keycloak/testsuite/crossdc/ActionTokenCrossDCTest.java).
+
+The cross DC requires setting a profile specifying used cache server (currently only Infinispan) by specifying
+`cache-server-infinispan` profile in maven.
+
+#### Run Cross-DC Tests from Maven
+
+Run the following command (adjust the test specification according to your needs):
+
+  `mvn -Pcache-server-infinispan -Dtest=*.crossdc.* -pl testsuite/integration-arquillian/tests/base test`
+
+_Someone using IntelliJ IDEA, please describe steps for that IDE_

testsuite/integration-arquillian/pom.xml

@@ -46,6 +46,7 @@
         <arquillian-drone.version>2.0.1.Final</arquillian-drone.version>
         <arquillian-graphene.version>2.1.0.Alpha3</arquillian-graphene.version>
         <arquillian-wildfly-container.version>2.1.0.Alpha2</arquillian-wildfly-container.version>
+        <arquillian-infinispan-container.version>1.2.0.Beta2</arquillian-infinispan-container.version>
         <version.shrinkwrap.resolvers>2.2.2</version.shrinkwrap.resolvers>
         <undertow-embedded.version>1.0.0.Alpha2</undertow-embedded.version>
 
@@ -87,6 +88,11 @@
                 <type>pom</type>
                 <scope>import</scope>
             </dependency>
+            <dependency>
+                <groupId>org.infinispan.arquillian.container</groupId>
+                <artifactId>infinispan-arquillian-impl</artifactId>
+                <version>${arquillian-infinispan-container.version}</version>
+            </dependency>
             <dependency>
                 <groupId>org.wildfly.arquillian</groupId>
                 <artifactId>wildfly-arquillian-container-managed</artifactId>

testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/arquillian/LoadBalancerController.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.arquillian;
+
+/**
+ *
+ * @author hmlnarik
+ */
+public interface LoadBalancerController {
+
+    void enableAllBackendNodes();
+
+    void disableAllBackendNodes();
+
+    void enableBackendNodeByName(String nodeName);
+
+    void disableBackendNodeByName(String nodeName);
+
+}

testsuite/integration-arquillian/servers/auth-server/undertow/pom.xml

@@ -70,6 +70,11 @@
             <groupId>org.keycloak</groupId>
             <artifactId>keycloak-undertow-adapter</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.keycloak.testsuite</groupId>
+            <artifactId>integration-arquillian-testsuite-providers</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.keycloak</groupId>
             <artifactId>keycloak-servlet-filter-adapter</artifactId>

testsuite/integration-arquillian/servers/auth-server/undertow/src/main/java/org/keycloak/testsuite/arquillian/undertow/KeycloakOnUndertow.java

@@ -48,6 +48,8 @@ import org.keycloak.services.filters.KeycloakSessionServletFilter;
 import org.keycloak.services.managers.ApplianceBootstrap;
 import org.keycloak.services.resources.KeycloakApplication;
 
+import org.keycloak.util.JsonSerialization;
+import java.io.IOException;
 import javax.servlet.DispatcherType;
 import javax.servlet.ServletException;
 
@@ -55,6 +57,7 @@ import java.lang.reflect.Field;
 import java.util.Collection;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.logging.Level;
 
 public class KeycloakOnUndertow implements DeployableContainer<KeycloakOnUndertowConfiguration> {
 
@@ -75,6 +78,14 @@ public class KeycloakOnUndertow implements DeployableContainer<KeycloakOnUnderto
         di.setContextPath("/auth");
         di.setDeploymentName("Keycloak");
         di.addInitParameter(KeycloakApplication.KEYCLOAK_EMBEDDED, "true");
+        if (configuration.getKeycloakConfigPropertyOverridesMap() != null) {
+            try {
+                di.addInitParameter(KeycloakApplication.SERVER_CONTEXT_CONFIG_PROPERTY_OVERRIDES,
+                  JsonSerialization.writeValueAsString(configuration.getKeycloakConfigPropertyOverridesMap()));
+            } catch (IOException ex) {
+                throw new RuntimeException(ex);
+            }
+        }
 
         di.setDefaultServletConfig(new DefaultServletConfig(true));
         di.addWelcomePage("theme/keycloak/welcome/resources/index.html");
@@ -176,19 +187,14 @@ public class KeycloakOnUndertow implements DeployableContainer<KeycloakOnUnderto
             log.info("Using route: " + configuration.getRoute());
         }
 
-        SetSystemProperty setRouteProperty = new SetSystemProperty(InfinispanConnectionProvider.JBOSS_NODE_NAME, configuration.getRoute());
+        DeploymentInfo di = createAuthServerDeploymentInfo();
-        try {
+        undertow.deploy(di);
-            DeploymentInfo di = createAuthServerDeploymentInfo();
+        ResteasyDeployment deployment = (ResteasyDeployment) di.getServletContextAttributes().get(ResteasyDeployment.class.getName());
-            undertow.deploy(di);
+        sessionFactory = ((KeycloakApplication) deployment.getApplication()).getSessionFactory();
-            ResteasyDeployment deployment = (ResteasyDeployment) di.getServletContextAttributes().get(ResteasyDeployment.class.getName());
-            sessionFactory = ((KeycloakApplication) deployment.getApplication()).getSessionFactory();
 
-            setupDevConfig();
+        setupDevConfig();
 
-            log.info("Auth server started in " + (System.currentTimeMillis() - start) + " ms\n");
+        log.info("Auth server started in " + (System.currentTimeMillis() - start) + " ms\n");
-        } finally {
-            setRouteProperty.revert();
-        }
     }
 
 

testsuite/integration-arquillian/servers/auth-server/undertow/src/main/java/org/keycloak/testsuite/arquillian/undertow/KeycloakOnUndertowConfiguration.java

@@ -17,6 +17,11 @@
 
 package org.keycloak.testsuite.arquillian.undertow;
 
+import org.keycloak.util.JsonSerialization;
+import com.fasterxml.jackson.core.type.TypeReference;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 import org.arquillian.undertow.UndertowContainerConfiguration;
 import org.jboss.arquillian.container.spi.ConfigurationException;
 import org.jboss.logging.Logger;
@@ -29,6 +34,8 @@ public class KeycloakOnUndertowConfiguration extends UndertowContainerConfigurat
     private String resourcesHome;
     private boolean remoteMode;
     private String route;
+    private String keycloakConfigPropertyOverrides;
+    private Map<String, String> keycloakConfigPropertyOverridesMap;
 
     private int bindHttpPortOffset = 0;
 
@@ -72,6 +79,18 @@ public class KeycloakOnUndertowConfiguration extends UndertowContainerConfigurat
         this.remoteMode = remoteMode;
     }
 
+    public String getKeycloakConfigPropertyOverrides() {
+        return keycloakConfigPropertyOverrides;
+    }
+
+    public void setKeycloakConfigPropertyOverrides(String keycloakConfigPropertyOverrides) {
+        this.keycloakConfigPropertyOverrides = keycloakConfigPropertyOverrides;
+    }
+
+    public Map<String, String> getKeycloakConfigPropertyOverridesMap() {
+        return keycloakConfigPropertyOverridesMap;
+    }
+
     @Override
     public void validate() throws ConfigurationException {
         super.validate();
@@ -80,6 +99,15 @@ public class KeycloakOnUndertowConfiguration extends UndertowContainerConfigurat
         int newPort = basePort + bindHttpPortOffset;
         setBindHttpPort(newPort);
         log.info("KeycloakOnUndertow will listen on port: " + newPort);
+        
+        if (this.keycloakConfigPropertyOverrides != null) {
+            try {
+                TypeReference<HashMap<String,Object>> typeRef = new TypeReference<HashMap<String,Object>>() {};
+                this.keycloakConfigPropertyOverridesMap = JsonSerialization.sysPropertiesAwareMapper.readValue(this.keycloakConfigPropertyOverrides, typeRef);
+            } catch (IOException ex) {
+                throw new ConfigurationException(ex);
+            }
+        }
 
         // TODO validate workerThreads
         

testsuite/integration-arquillian/servers/auth-server/undertow/src/main/java/org/keycloak/testsuite/arquillian/undertow/SetSystemProperty.java

@@ -17,13 +17,15 @@
 
 package org.keycloak.testsuite.arquillian.undertow;
 
+import java.io.Closeable;
+
 /**
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
-class SetSystemProperty {
+class SetSystemProperty implements Closeable {
 
-    private String name;
+    private final String name;
-    private String oldValue;
+    private final String oldValue;
 
     public SetSystemProperty(String name, String value) {
         this.name = name;
@@ -50,4 +52,9 @@ class SetSystemProperty {
         }
     }
 
+    @Override
+    public void close() {
+        revert();
+    }
+
 }

testsuite/integration-arquillian/servers/auth-server/undertow/src/main/java/org/keycloak/testsuite/arquillian/undertow/lb/SimpleUndertowLoadBalancer.java

@@ -18,7 +18,6 @@
 package org.keycloak.testsuite.arquillian.undertow.lb;
 
 import java.net.URI;
-import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
@@ -36,6 +35,8 @@ import io.undertow.util.AttachmentKey;
 import io.undertow.util.Headers;
 import org.jboss.logging.Logger;
 import org.keycloak.services.managers.AuthenticationSessionManager;
+import java.util.LinkedHashMap;
+import java.util.StringTokenizer;
 
 /**
  * Loadbalancer on embedded undertow. Supports sticky session over "AUTH_SESSION_ID" cookie and failover to different node when sticky node not available.
@@ -53,8 +54,9 @@ public class SimpleUndertowLoadBalancer {
 
     private final String host;
     private final int port;
-    private final String nodesString;
+    private final Map<String, URI> backendNodes;
     private Undertow undertow;
+    private LoadBalancingProxyClient lb;
 
 
     public static void main(String[] args) throws Exception {
@@ -77,15 +79,14 @@ public class SimpleUndertowLoadBalancer {
     public SimpleUndertowLoadBalancer(String host, int port, String nodesString) {
         this.host = host;
         this.port = port;
-        this.nodesString = nodesString;
+        this.backendNodes = parseNodes(nodesString);
-        log.infof("Keycloak nodes: %s", nodesString);
+        log.infof("Keycloak nodes: %s", backendNodes);
     }
 
 
     public void start() {
-        Map<String, String> nodes = parseNodes(nodesString);
         try {
-            HttpHandler proxyHandler = createHandler(nodes);
+            HttpHandler proxyHandler = createHandler();
 
             undertow = Undertow.builder()
                     .addHttpListener(port, host)
@@ -104,24 +105,51 @@ public class SimpleUndertowLoadBalancer {
         undertow.stop();
     }
 
+    public void enableAllBackendNodes() {
+        backendNodes.forEach((route, uri) -> {
+            lb.removeHost(uri);
+            lb.addHost(uri, route);
+        });
+    }
 
-    static Map<String, String> parseNodes(String nodes) {
+    public void disableAllBackendNodes() {
-        String[] nodesArray = nodes.split(",");
+        log.debugf("Load balancer: disabling all nodes");
-        Map<String, String> result = new HashMap<>();
+        backendNodes.values().forEach(lb::removeHost);
+    }
 
-        for (String nodeStr : nodesArray) {
+    public void enableBackendNodeByName(String nodeName) {
-            String[] node = nodeStr.trim().split("=");
+        URI uri = backendNodes.get(nodeName);
-            if (node.length != 2) {
+        if (uri == null) {
-                throw new IllegalArgumentException("Illegal node format in the configuration: " + nodeStr);
+            throw new IllegalArgumentException("Invalid node: " + nodeName);
-            }
+        }
-            result.put(node[0].trim(), node[1].trim());
+        log.debugf("Load balancer: enabling node %s", nodeName);
+        lb.addHost(uri, nodeName);
+    }
+
+    public void disableBackendNodeByName(String nodeName) {
+        URI uri = backendNodes.get(nodeName);
+        if (uri == null) {
+            throw new IllegalArgumentException("Invalid node: " + nodeName);
+        }
+        log.debugf("Load balancer: disabling node %s", nodeName);
+        lb.removeHost(uri);
+    }
+
+    static Map<String, URI> parseNodes(String nodes) {
+        StringTokenizer st = new StringTokenizer(nodes, ",");
+        Map<String, URI> result = new LinkedHashMap<>();
+
+        while (st.hasMoreElements()) {
+            String nodeStr = st.nextToken();
+            String[] node = nodeStr.trim().split("=", 2);
+            result.put(node[0].trim(), URI.create(node[1].trim()));
         }
 
         return result;
     }
 
 
-    private HttpHandler createHandler(Map<String, String> backendNodes) throws Exception {
+    private HttpHandler createHandler() throws Exception {
 
         // TODO: configurable options if needed
         String sessionCookieNames = AuthenticationSessionManager.AUTH_SESSION_ID;
@@ -133,15 +161,7 @@ public class SimpleUndertowLoadBalancer {
         int connectionIdleTimeout = 60;
         int maxRetryAttempts = backendNodes.size() - 1;
 
-        final LoadBalancingProxyClient lb = new CustomLoadBalancingClient(new ExclusivityChecker() {
+        lb = new CustomLoadBalancingClient(exchange -> exchange.getRequestHeaders().contains(Headers.UPGRADE), maxRetryAttempts)
-
-            @Override
-            public boolean isExclusivityRequired(HttpServerExchange exchange) {
-                //we always create a new connection for upgrade requests
-                return exchange.getRequestHeaders().contains(Headers.UPGRADE);
-            }
-
-        }, maxRetryAttempts)
                 .setConnectionsPerThread(connectionsPerThread)
                 .setMaxQueueSize(requestQueueSize)
                 .setSoftMaxConnectionsPerThread(cachedConnectionsPerThread)
@@ -152,13 +172,10 @@ public class SimpleUndertowLoadBalancer {
             lb.addSessionCookieName(id);
         }
 
-        for (Map.Entry<String, String> node : backendNodes.entrySet()) {
+        backendNodes.forEach((route, uri) -> {
-            String route = node.getKey();
-            URI uri = new URI(node.getValue());
-
             lb.addHost(uri, route);
-            log.infof("Added host: %s, route: %s", uri.toString(), route);
+            log.debugf("Added host: %s, route: %s", uri.toString(), route);
-        }
+        });
 
         ProxyHandler handler = new ProxyHandler(lb, maxTime, ResponseCodeHandler.HANDLE_404);
         return handler;

testsuite/integration-arquillian/servers/auth-server/undertow/src/main/java/org/keycloak/testsuite/arquillian/undertow/lb/SimpleUndertowLoadBalancerConfiguration.java

@@ -19,13 +19,17 @@ package org.keycloak.testsuite.arquillian.undertow.lb;
 
 import org.arquillian.undertow.UndertowContainerConfiguration;
 import org.jboss.arquillian.container.spi.ConfigurationException;
+import org.jboss.logging.Logger;
 
 /**
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
 public class SimpleUndertowLoadBalancerConfiguration extends UndertowContainerConfiguration {
 
+    protected static final Logger log = Logger.getLogger(SimpleUndertowLoadBalancerConfiguration.class);
+
     private String nodes = SimpleUndertowLoadBalancer.DEFAULT_NODES;
+    private int bindHttpPortOffset = 0;
 
     public String getNodes() {
         return nodes;
@@ -35,6 +39,14 @@ public class SimpleUndertowLoadBalancerConfiguration extends UndertowContainerCo
         this.nodes = nodes;
     }
 
+    public int getBindHttpPortOffset() {
+        return bindHttpPortOffset;
+    }
+
+    public void setBindHttpPortOffset(int bindHttpPortOffset) {
+        this.bindHttpPortOffset = bindHttpPortOffset;
+    }
+
     @Override
     public void validate() throws ConfigurationException {
         super.validate();
@@ -44,5 +56,11 @@ public class SimpleUndertowLoadBalancerConfiguration extends UndertowContainerCo
         } catch (Exception e) {
             throw new ConfigurationException(e);
         }
+
+        int basePort = getBindHttpPort();
+        int newPort = basePort + bindHttpPortOffset;
+        setBindHttpPort(newPort);
+        log.info("SimpleUndertowLoadBalancer will listen on port: " + newPort);
+
     }
 }

testsuite/integration-arquillian/servers/auth-server/undertow/src/main/java/org/keycloak/testsuite/arquillian/undertow/lb/SimpleUndertowLoadBalancerContainer.java

@@ -25,13 +25,14 @@ import org.jboss.arquillian.container.spi.client.protocol.metadata.ProtocolMetaD
 import org.jboss.logging.Logger;
 import org.jboss.shrinkwrap.api.Archive;
 import org.jboss.shrinkwrap.descriptor.api.Descriptor;
+import org.keycloak.testsuite.arquillian.LoadBalancerController;
 
 /**
  * Arquillian container over {@link SimpleUndertowLoadBalancer}
  *
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
-public class SimpleUndertowLoadBalancerContainer implements DeployableContainer<SimpleUndertowLoadBalancerConfiguration> {
+public class SimpleUndertowLoadBalancerContainer implements DeployableContainer<SimpleUndertowLoadBalancerConfiguration>, LoadBalancerController {
 
     private static final Logger log = Logger.getLogger(SimpleUndertowLoadBalancerContainer.class);
 
@@ -84,4 +85,24 @@ public class SimpleUndertowLoadBalancerContainer implements DeployableContainer<
     public void undeploy(Descriptor descriptor) throws DeploymentException {
         throw new UnsupportedOperationException("Not implemented");
     }
+
+    @Override
+    public void enableAllBackendNodes() {
+        this.container.enableAllBackendNodes();
+    }
+
+    @Override
+    public void disableAllBackendNodes() {
+        this.container.disableAllBackendNodes();
+    }
+
+    @Override
+    public void enableBackendNodeByName(String nodeName) {
+        this.container.enableBackendNodeByName(nodeName);
+    }
+
+    @Override
+    public void disableBackendNodeByName(String nodeName) {
+        this.container.disableBackendNodeByName(nodeName);
+    }
 }

testsuite/integration-arquillian/servers/cache-server/jboss/assembly.xml

@@ -0,0 +1,46 @@
+<!--
+  ~ Copyright 2016 Red Hat, Inc. and/or its affiliates
+  ~ and other contributors as indicated by the @author tags.
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~ http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<assembly>
+    
+    <id>${cache.server.jboss}</id>
+    
+    <formats>
+        <format>zip</format>
+    </formats>
+
+    <includeBaseDirectory>false</includeBaseDirectory>
+
+    <fileSets>
+        <fileSet>
+            <directory>${cache.server.jboss.home}</directory>
+            <outputDirectory>cache-server-${cache.server}</outputDirectory>
+            <excludes>
+                <exclude>**/*.sh</exclude>
+            </excludes>
+        </fileSet>
+        <fileSet>
+            <directory>${cache.server.jboss.home}</directory>
+            <outputDirectory>cache-server-${cache.server}</outputDirectory>
+            <includes>
+                <include>**/*.sh</include>
+            </includes>
+            <fileMode>0755</fileMode>
+        </fileSet>
+    </fileSets>
+
+</assembly>

testsuite/integration-arquillian/servers/cache-server/jboss/common/add-keycloak-remote-store.xsl

@@ -0,0 +1,42 @@
+<!--
+  ~ Copyright 2016 Red Hat, Inc. and/or its affiliates
+  ~ and other contributors as indicated by the @author tags.
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~ http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+                xmlns:xalan="http://xml.apache.org/xalan"
+                version="2.0"
+                exclude-result-prefixes="xalan">
+
+    <xsl:output method="xml" version="1.0" encoding="UTF-8" indent="yes" xalan:indent-amount="4" standalone="no"/>
+    <xsl:strip-space elements="*"/>
+
+    <xsl:variable name="nsCacheServer" select="'urn:infinispan:server:core:'"/>
+
+    <xsl:template match="//*[local-name()='subsystem' and starts-with(namespace-uri(), $nsCacheServer)]
+                        /*[local-name()='cache-container' and starts-with(namespace-uri(), $nsCacheServer) and @name='local']">
+        <xsl:copy>
+            <xsl:apply-templates select="@* | node()" />
+            <local-cache name="work" start="EAGER" batching="false" />
+        </xsl:copy>
+    </xsl:template>
+
+    <xsl:template match="@*|node()">
+        <xsl:copy>
+            <xsl:apply-templates select="@*|node()" />
+        </xsl:copy>
+    </xsl:template>
+
+</xsl:stylesheet>

testsuite/integration-arquillian/servers/cache-server/jboss/common/io.xsl

@@ -0,0 +1,40 @@
+<!--
+~ Copyright 2016 Red Hat, Inc. and/or its affiliates
+~ and other contributors as indicated by the @author tags.
+~
+~ Licensed under the Apache License, Version 2.0 (the "License");
+~ you may not use this file except in compliance with the License.
+~ You may obtain a copy of the License at
+~
+~ http://www.apache.org/licenses/LICENSE-2.0
+~
+~ Unless required by applicable law or agreed to in writing, software
+~ distributed under the License is distributed on an "AS IS" BASIS,
+~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~ See the License for the specific language governing permissions and
+~ limitations under the License.
+-->
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+                xmlns:xalan="http://xml.apache.org/xalan"
+                version="2.0"
+                exclude-result-prefixes="xalan">
+
+    <xsl:output method="xml" version="1.0" encoding="UTF-8" indent="yes" xalan:indent-amount="4" standalone="no"/>
+    <xsl:strip-space elements="*"/>
+
+    <xsl:param name="worker.io-threads" select="'16'"/>
+    <xsl:param name="worker.task-max-threads" select="'128'"/>
+    
+    <!--set worker threads-->
+    <xsl:template match="//*[local-name()='worker' and @name='default']">
+        <worker name="default" io-threads="{$worker.io-threads}" task-max-threads="{$worker.task-max-threads}" />
+    </xsl:template>
+
+    <xsl:template match="@*|node()">
+        <xsl:copy>
+            <xsl:apply-templates select="@*|node()" />
+        </xsl:copy>
+    </xsl:template>
+
+</xsl:stylesheet>

testsuite/integration-arquillian/servers/cache-server/jboss/infinispan/pom.xml

@@ -0,0 +1,46 @@
+<?xml version="1.0"?>
+<!--
+~ Copyright 2016 Red Hat, Inc. and/or its affiliates
+~ and other contributors as indicated by the @author tags.
+~
+~ Licensed under the Apache License, Version 2.0 (the "License");
+~ you may not use this file except in compliance with the License.
+~ You may obtain a copy of the License at
+~
+~ http://www.apache.org/licenses/LICENSE-2.0
+~
+~ Unless required by applicable law or agreed to in writing, software
+~ distributed under the License is distributed on an "AS IS" BASIS,
+~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~ See the License for the specific language governing permissions and
+~ limitations under the License.
+-->
+
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+    <parent>
+        <groupId>org.keycloak.testsuite</groupId>
+        <artifactId>integration-arquillian-servers-cache-server-jboss</artifactId>
+        <version>3.2.0.CR1-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>integration-arquillian-servers-cache-server-infinispan</artifactId>
+    <packaging>pom</packaging>
+    <name>Cache Server - JBoss - Infinispan</name>
+    
+    <properties>
+        <cache.server>infinispan</cache.server>
+        <cache.server.container>cache-server-${cache.server}</cache.server.container>
+        <cache.server.home>${containers.home}/${cache.server.container}</cache.server.home>
+        
+        <cache.server.jboss.groupId>org.infinispan.server</cache.server.jboss.groupId>
+        <cache.server.jboss.artifactId>infinispan-server</cache.server.jboss.artifactId>
+        <cache.server.jboss.version>${infinispan.version}</cache.server.jboss.version>
+        <cache.server.jboss.unpacked.folder.name>${cache.server.jboss.artifactId}-${infinispan.version}</cache.server.jboss.unpacked.folder.name>
+        
+        <cache.server.worker.io-threads>${cache.default.worker.io-threads}</cache.server.worker.io-threads>
+        <cache.server.worker.task-max-threads>${cache.default.worker.task-max-threads}</cache.server.worker.task-max-threads>
+    </properties>
+
+</project>

testsuite/integration-arquillian/servers/cache-server/jboss/pom.xml

@@ -0,0 +1,210 @@
+<?xml version="1.0"?>
+<!--
+~ Copyright 2016 Red Hat, Inc. and/or its affiliates
+~ and other contributors as indicated by the @author tags.
+~
+~ Licensed under the Apache License, Version 2.0 (the "License");
+~ you may not use this file except in compliance with the License.
+~ You may obtain a copy of the License at
+~
+~ http://www.apache.org/licenses/LICENSE-2.0
+~
+~ Unless required by applicable law or agreed to in writing, software
+~ distributed under the License is distributed on an "AS IS" BASIS,
+~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~ See the License for the specific language governing permissions and
+~ limitations under the License.
+-->
+
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+    <parent>
+        <groupId>org.keycloak.testsuite</groupId>
+        <artifactId>integration-arquillian-servers-cache-server</artifactId>
+        <version>3.2.0.CR1-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>integration-arquillian-servers-cache-server-jboss</artifactId>
+    <packaging>pom</packaging>
+    <name>Cache Server - JBoss Family</name>
+
+    <properties>
+        <common.resources>${project.parent.basedir}/common</common.resources>
+        <assembly.xml>${project.parent.basedir}/assembly.xml</assembly.xml>
+        <cache.server.jboss.home>${containers.home}/${cache.server.jboss.unpacked.folder.name}</cache.server.jboss.home>
+        <security.xslt>security.xsl</security.xslt>
+    </properties>
+
+    <profiles>
+        
+        <profile>
+            <id>cache-server-jboss-submodules</id>
+            <activation>
+                <file>
+                    <exists>src</exists>
+                </file>
+            </activation>
+            <build>
+                <plugins>
+                    
+                    <plugin>
+                        <artifactId>maven-enforcer-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <goals>
+                                    <goal>enforce</goal>
+                                </goals>
+                                <configuration>
+                                    <rules>
+                                        <requireProperty>
+                                            <property>cache.server</property>
+                                            <property>cache.server.jboss.groupId</property>
+                                            <property>cache.server.jboss.artifactId</property>
+                                            <property>cache.server.jboss.version</property>
+                                            <property>cache.server.jboss.unpacked.folder.name</property>
+                                        </requireProperty>
+                                    </rules>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                    
+                    <plugin>
+                        <artifactId>maven-dependency-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <id>unpack-cache-server</id>
+                                <phase>generate-resources</phase>
+                                <goals>
+                                    <goal>unpack</goal>
+                                </goals>
+                                <configuration>
+                                    <artifactItems>
+                                        <artifactItem>
+                                            <groupId>${cache.server.jboss.groupId}</groupId>
+                                            <artifactId>${cache.server.jboss.artifactId}</artifactId>
+                                            <version>${cache.server.jboss.version}</version>
+                                            <type>zip</type>
+                                            <classifier>bin</classifier>
+                                            <outputDirectory>${containers.home}</outputDirectory>
+                                        </artifactItem>
+                                    </artifactItems>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+            
+                    <plugin>
+                        <groupId>org.codehaus.mojo</groupId>
+                        <artifactId>xml-maven-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <id>configure-adapter-debug-log</id>
+                                <phase>process-test-resources</phase>
+                                <goals>
+                                    <goal>transform</goal>
+                                </goals>
+                                <configuration>
+                                    <transformationSets>
+                                        <transformationSet>
+                                            <dir>${cache.server.jboss.home}/standalone/configuration</dir>
+                                            <includes>
+                                                <include>standalone.xml</include>
+                                            </includes>
+                                            <stylesheet>${common.resources}/add-keycloak-remote-store.xsl</stylesheet>
+                                            <outputDir>${cache.server.jboss.home}/standalone/configuration</outputDir>
+                                        </transformationSet>
+                                    </transformationSets>
+                                </configuration>
+                            </execution>
+                            <execution>
+                                <id>io-worker-threads</id>
+                                <phase>process-resources</phase>
+                                <goals>
+                                    <goal>transform</goal>
+                                </goals>
+                                <configuration>
+                                    <transformationSets>
+                                        <transformationSet>
+                                            <dir>${cache.server.jboss.home}/standalone/configuration</dir>
+                                            <includes>
+                                                <include>standalone.xml</include>
+                                                <include>standalone-ha.xml</include>
+                                            </includes>
+                                            <stylesheet>${common.resources}/io.xsl</stylesheet>
+                                            <outputDir>${cache.server.jboss.home}/standalone/configuration</outputDir>
+                                            <parameters>
+                                                <parameter>
+                                                    <name>worker.io-threads</name>
+                                                    <value>${cache.server.worker.io-threads}</value>
+                                                </parameter>
+                                                <parameter>
+                                                    <name>worker.task-max-threads</name>
+                                                    <value>${cache.server.worker.task-max-threads}</value>
+                                                </parameter>
+                                            </parameters>
+                                        </transformationSet>
+                                    </transformationSets>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                    
+                    <plugin>
+                        <artifactId>maven-resources-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <id>enable-jboss-mgmt-admin</id>
+                                <phase>process-resources</phase>
+                                <goals>
+                                    <goal>copy-resources</goal>
+                                </goals>
+                                <configuration>
+                                    <outputDirectory>${cache.server.jboss.home}/standalone/configuration</outputDirectory>
+                                    <resources>
+                                        <resource>
+                                            <directory>${common.resources}</directory>
+                                            <includes>
+                                                <include>mgmt-users.properties</include>
+                                            </includes>
+                                        </resource>
+                                    </resources>
+                                    <overwrite>true</overwrite>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+
+                    <plugin>
+                        <artifactId>maven-assembly-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <id>create-zip</id>
+                                <phase>package</phase>
+                                <goals>
+                                    <goal>single</goal>
+                                </goals>
+                                <configuration>
+                                    <descriptors>
+                                        <descriptor>${assembly.xml}</descriptor>
+                                    </descriptors>
+                                    <appendAssemblyId>false</appendAssemblyId>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                    
+                </plugins>
+            </build>
+        </profile>
+        
+        <profile>
+            <id>cache-server-infinispan</id>
+            <modules>
+                <module>infinispan</module>
+            </modules>
+        </profile>
+    </profiles>
+
+</project>

testsuite/integration-arquillian/servers/cache-server/pom.xml

@@ -0,0 +1,41 @@
+<?xml version="1.0"?>
+<!--
+~ Copyright 2016 Red Hat, Inc. and/or its affiliates
+~ and other contributors as indicated by the @author tags.
+~
+~ Licensed under the Apache License, Version 2.0 (the "License");
+~ you may not use this file except in compliance with the License.
+~ You may obtain a copy of the License at
+~
+~ http://www.apache.org/licenses/LICENSE-2.0
+~
+~ Unless required by applicable law or agreed to in writing, software
+~ distributed under the License is distributed on an "AS IS" BASIS,
+~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~ See the License for the specific language governing permissions and
+~ limitations under the License.
+-->
+
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+    <parent>
+        <groupId>org.keycloak.testsuite</groupId>
+        <artifactId>integration-arquillian-servers</artifactId>
+        <version>3.2.0.CR1-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>integration-arquillian-servers-cache-server</artifactId>
+    <packaging>pom</packaging>
+    <name>Cache Server</name>
+
+    <properties>
+        <auth.server.worker.io-threads>${jboss.default.worker.io-threads}</auth.server.worker.io-threads>
+        <auth.server.worker.task-max-threads>${jboss.default.worker.task-max-threads}</auth.server.worker.task-max-threads>
+    </properties>
+
+    <modules>
+        <module>jboss</module>
+    </modules>
+
+</project>

testsuite/integration-arquillian/servers/pom.xml

@@ -46,13 +46,20 @@
         <!--<fuse62.version>6.2.0.redhat-133</fuse62.version>-->
         <fuse62.version>6.2.1.redhat-084</fuse62.version>
         
+        <!-- cache server versions -->
+        <infinispan.version>9.0.1.Final</infinispan.version>
+        
         <jboss.default.worker.io-threads>16</jboss.default.worker.io-threads>
         <jboss.default.worker.task-max-threads>128</jboss.default.worker.task-max-threads>
+
+        <cache.default.worker.io-threads>2</cache.default.worker.io-threads>
+        <cache.default.worker.task-max-threads>4</cache.default.worker.task-max-threads>
     </properties>
 
     <modules>
         <module>auth-server</module>
         <module>app-server</module>
+        <module>cache-server</module>
     </modules>
 
     <profiles>

testsuite/integration-arquillian/tests/base/pom.xml

@@ -40,6 +40,7 @@
         <exclude.client>-</exclude.client>
         <!--exclude cluster tests by default, enabled by 'auth-server-*-cluster' profiles in tests/pom.xml-->
         <exclude.cluster>**/cluster/**/*Test.java</exclude.cluster>
+        <exclude.crossdc>**/crossdc/**/*Test.java</exclude.crossdc>
         <!-- exclude x509 tests by default, enabled by 'ssl' profile -->
         <exclude.x509>**/x509/*Test.java</exclude.x509>
         <!-- exclude undertow adapter tests. They can be added by -Dtest=org.keycloak.testsuite.adapter.undertow.**.*Test -->
@@ -125,6 +126,7 @@
                         <exclude>${exclude.account}</exclude>
                         <exclude>${exclude.client}</exclude>
                         <exclude>${exclude.cluster}</exclude>
+                        <exclude>${exclude.crossdc</exclude>
                         <exclude>${exclude.undertow.adapter}</exclude>
                         <exclude>${exclude.x509}</exclude>
                     </excludes>

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/AuthServerTestEnricher.java

@@ -16,7 +16,6 @@
  */
 package org.keycloak.testsuite.arquillian;
 
-import org.jboss.arquillian.container.spi.Container;
 import org.jboss.arquillian.container.spi.ContainerRegistry;
 import org.jboss.arquillian.container.spi.event.StartContainer;
 import org.jboss.arquillian.container.spi.event.StartSuiteContainers;
@@ -41,10 +40,11 @@ import org.keycloak.testsuite.util.OAuthClient;
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
-import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Objects;
 import java.util.Set;
 
+import java.util.stream.Collectors;
 import javax.ws.rs.NotFoundException;
 
 /**
@@ -68,8 +68,18 @@ public class AuthServerTestEnricher {
     private static final String AUTH_SERVER_CONTAINER_PROPERTY = "auth.server.container";
     public static final String AUTH_SERVER_CONTAINER = System.getProperty(AUTH_SERVER_CONTAINER_PROPERTY, AUTH_SERVER_CONTAINER_DEFAULT);
 
+    private static final String AUTH_SERVER_BACKEND_DEFAULT = AUTH_SERVER_CONTAINER + "-backend";
+    private static final String AUTH_SERVER_BACKEND_PROPERTY = "auth.server.backend";
+    public static final String AUTH_SERVER_BACKEND = System.getProperty(AUTH_SERVER_BACKEND_PROPERTY, AUTH_SERVER_BACKEND_DEFAULT);
+
+    private static final String AUTH_SERVER_BALANCER_DEFAULT = "auth-server-balancer";
+    private static final String AUTH_SERVER_BALANCER_PROPERTY = "auth.server.balancer";
+    public static final String AUTH_SERVER_BALANCER = System.getProperty(AUTH_SERVER_BALANCER_PROPERTY, AUTH_SERVER_BALANCER_DEFAULT);
+
     private static final String AUTH_SERVER_CLUSTER_PROPERTY = "auth.server.cluster";
     public static final boolean AUTH_SERVER_CLUSTER = Boolean.parseBoolean(System.getProperty(AUTH_SERVER_CLUSTER_PROPERTY, "false"));
+    private static final String AUTH_SERVER_CROSS_DC_PROPERTY = "auth.server.crossdc";
+    public static final boolean AUTH_SERVER_CROSS_DC = Boolean.parseBoolean(System.getProperty(AUTH_SERVER_CROSS_DC_PROPERTY, "false"));
 
     private static final boolean AUTH_SERVER_UNDERTOW_CLUSTER = Boolean.parseBoolean(System.getProperty("auth.server.undertow.cluster", "false"));
 
@@ -106,46 +116,82 @@ public class AuthServerTestEnricher {
     }
 
     public void initializeSuiteContext(@Observes(precedence = 2) BeforeSuite event) {
-
+        Set<ContainerInfo> containers = containerRegistry.get().getContainers().stream()
-        Set<ContainerInfo> containers = new LinkedHashSet<>();
+          .map(ContainerInfo::new)
-        for (Container c : containerRegistry.get().getContainers()) {
+          .collect(Collectors.toSet());
-            containers.add(new ContainerInfo(c));
-        }
 
         suiteContext = new SuiteContext(containers);
 
-        String authServerFrontend = null;
+        if (AUTH_SERVER_CROSS_DC) {
+            // if cross-dc mode enabled, load-balancer is the frontend of datacenter cluster
+            containers.stream()
+              .filter(c -> c.getQualifier().startsWith(AUTH_SERVER_BALANCER + "-cross-dc"))
+              .forEach(c -> {
+                String portOffsetString = c.getArquillianContainer().getContainerConfiguration().getContainerProperties().getOrDefault("bindHttpPortOffset", "0");
+                String dcString = c.getArquillianContainer().getContainerConfiguration().getContainerProperties().getOrDefault("dataCenter", "0");
+                updateWithAuthServerInfo(c, Integer.valueOf(portOffsetString));
+                suiteContext.addAuthServerInfo(Integer.valueOf(dcString), c);
+              });
 
-        if (AUTH_SERVER_CLUSTER) {
+            if (suiteContext.getDcAuthServerInfo().isEmpty()) {
+                throw new IllegalStateException("Not found frontend container (load balancer): " + AUTH_SERVER_BALANCER);
+            }
+            if (suiteContext.getDcAuthServerInfo().stream().anyMatch(Objects::isNull)) {
+                throw new IllegalStateException("Frontend container (load balancer) misconfiguration");
+            }
+
+            containers.stream()
+              .filter(c -> c.getQualifier().startsWith(AUTH_SERVER_CONTAINER + "-cross-dc-"))
+              .forEach(c -> {
+                String portOffsetString = c.getArquillianContainer().getContainerConfiguration().getContainerProperties().getOrDefault("bindHttpPortOffset", "0");
+                String dcString = c.getArquillianContainer().getContainerConfiguration().getContainerProperties().getOrDefault("dataCenter", "0");
+                updateWithAuthServerInfo(c, Integer.valueOf(portOffsetString));
+                suiteContext.addAuthServerBackendsInfo(Integer.valueOf(dcString), c);
+              });
+
+            if (suiteContext.getDcAuthServerInfo().isEmpty()) {
+                throw new RuntimeException(String.format("No auth server container matching '%s' found in arquillian.xml.", AUTH_SERVER_BACKEND));
+            }
+            if (suiteContext.getDcAuthServerBackendsInfo().stream().anyMatch(Objects::isNull)) {
+                throw new IllegalStateException("Frontend container (load balancer) misconfiguration");
+            }
+            if (suiteContext.getDcAuthServerBackendsInfo().stream().anyMatch(List::isEmpty)) {
+                throw new RuntimeException(String.format("Some data center has no auth server container matching '%s' defined in arquillian.xml.", AUTH_SERVER_BACKEND));
+            }
+
+            log.info("Using frontend containers: " + this.suiteContext.getDcAuthServerInfo().stream()
+              .map(ContainerInfo::getQualifier)
+              .collect(Collectors.joining(", ")));
+        } else if (AUTH_SERVER_CLUSTER) {
             // if cluster mode enabled, load-balancer is the frontend
-            for (ContainerInfo c : containers) {
+            ContainerInfo container = containers.stream()
-                if (c.getQualifier().startsWith("auth-server-balancer")) {
+              .filter(c -> c.getQualifier().startsWith(AUTH_SERVER_BALANCER))
-                    authServerFrontend = c.getQualifier();
+              .findAny()
-                }
+              .orElseThrow(() -> new IllegalStateException("Not found frontend container: " + AUTH_SERVER_BALANCER));
+            updateWithAuthServerInfo(container);
+            suiteContext.setAuthServerInfo(container);
+
+            containers.stream()
+              .filter(c -> c.getQualifier().startsWith(AUTH_SERVER_BACKEND))
+              .forEach(c -> {
+                String portOffsetString = c.getArquillianContainer().getContainerConfiguration().getContainerProperties().getOrDefault("bindHttpPortOffset", "0");
+                updateWithAuthServerInfo(c, Integer.valueOf(portOffsetString));
+                suiteContext.addAuthServerBackendsInfo(0, c);
+              });
+
+            if (suiteContext.getAuthServerBackendsInfo().isEmpty()) {
+                throw new RuntimeException(String.format("No auth server container matching '%s' found in arquillian.xml.", AUTH_SERVER_BACKEND));
             }
 
-            if (authServerFrontend != null) {
+            log.info("Using frontend container: " + container.getQualifier());
-                log.info("Using frontend container: " + authServerFrontend);
-            } else {
-                throw new IllegalStateException("Not found frontend container");
-            }
         } else {
-            authServerFrontend = AUTH_SERVER_CONTAINER; // single-node mode
+            // frontend-only
-        }
+            ContainerInfo container = containers.stream()
-
+              .filter(c -> c.getQualifier().startsWith(AUTH_SERVER_CONTAINER))
-        String authServerBackend = AUTH_SERVER_CONTAINER + "-backend";
+              .findAny()
-        int backends = 0;
+              .orElseThrow(() -> new IllegalStateException("Not found frontend container: " + AUTH_SERVER_CONTAINER));
-        for (ContainerInfo container : suiteContext.getContainers()) {
+            updateWithAuthServerInfo(container);
-            // frontend
+            suiteContext.setAuthServerInfo(container);
-            if (container.getQualifier().equals(authServerFrontend)) {
-                updateWithAuthServerInfo(container);
-                suiteContext.setAuthServerInfo(container);
-            }
-            // backends
-            if (AUTH_SERVER_CLUSTER && container.getQualifier().startsWith(authServerBackend)) {
-                updateWithAuthServerInfo(container, ++backends);
-                suiteContext.getAuthServerBackendsInfo().add(container);
-            }
         }
 
         // Setup with 2 undertow backend nodes and no loadbalancer.
@@ -153,14 +199,6 @@ public class AuthServerTestEnricher {
 //            suiteContext.setAuthServerInfo(suiteContext.getAuthServerBackendsInfo().get(0));
 //        }
 
-        // validate auth server setup
-        if (suiteContext.getAuthServerInfo() == null) {
-            throw new RuntimeException(String.format("No auth server container matching '%s' found in arquillian.xml.", authServerFrontend));
-        }
-        if (AUTH_SERVER_CLUSTER && suiteContext.getAuthServerBackendsInfo().isEmpty()) {
-            throw new RuntimeException(String.format("No auth server container matching '%sN' found in arquillian.xml.", authServerBackend));
-        }
-
         if (START_MIGRATION_CONTAINER) {
             // init migratedAuthServerInfo
             for (ContainerInfo container : suiteContext.getContainers()) {

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/ContainerInfo.java

@@ -5,6 +5,8 @@ import org.jboss.arquillian.container.spi.Container;
 import java.net.URL;
 import java.util.Map;
 import java.util.Objects;
+import java.util.stream.Stream;
+import org.jboss.arquillian.container.spi.Container.State;
 
 /**
  *
@@ -97,4 +99,12 @@ public class ContainerInfo {
                 other.arquillianContainer.getContainerConfiguration().getContainerName());
     }
 
+    public boolean isStarted() {
+        return arquillianContainer.getState() == State.STARTED;
+    }
+
+    public boolean isManual() {
+        return Objects.equals(arquillianContainer.getContainerConfiguration().getMode(), "manual");
+    }
+
 }

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/KeycloakArquillianExtension.java

@@ -35,6 +35,7 @@ import org.keycloak.testsuite.arquillian.h2.H2TestEnricher;
 import org.keycloak.testsuite.arquillian.karaf.CustomKarafContainer;
 import org.keycloak.testsuite.arquillian.migration.MigrationTestExecutionDecider;
 import org.keycloak.testsuite.arquillian.provider.AdminClientProvider;
+import org.keycloak.testsuite.arquillian.provider.LoadBalancerControllerProvider;
 import org.keycloak.testsuite.arquillian.provider.OAuthClientProvider;
 import org.keycloak.testsuite.arquillian.provider.SuiteContextProvider;
 import org.keycloak.testsuite.arquillian.provider.TestContextProvider;
@@ -57,7 +58,8 @@ public class KeycloakArquillianExtension implements LoadableExtension {
                 .service(ResourceProvider.class, SuiteContextProvider.class)
                 .service(ResourceProvider.class, TestContextProvider.class)
                 .service(ResourceProvider.class, AdminClientProvider.class)
-                .service(ResourceProvider.class, OAuthClientProvider.class);
+                .service(ResourceProvider.class, OAuthClientProvider.class)
+                .service(ResourceProvider.class, LoadBalancerControllerProvider.class);
 
         builder
                 .service(DeploymentScenarioGenerator.class, DeploymentTargetModifier.class)

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/SuiteContext.java

@@ -24,6 +24,7 @@ import java.util.Set;
 
 import org.keycloak.testsuite.arquillian.migration.MigrationContext;
 
+import java.util.LinkedList;
 import static org.keycloak.testsuite.util.MailServerConfiguration.FROM;
 import static org.keycloak.testsuite.util.MailServerConfiguration.HOST;
 import static org.keycloak.testsuite.util.MailServerConfiguration.PORT;
@@ -36,8 +37,8 @@ public final class SuiteContext {
 
     private final Set<ContainerInfo> container;
 
-    private ContainerInfo authServerInfo;
+    private List<ContainerInfo> authServerInfo = new LinkedList<>();
-    private final List<ContainerInfo> authServerBackendsInfo = new ArrayList<>();
+    private final List<List<ContainerInfo>> authServerBackendsInfo = new ArrayList<>();
 
     private ContainerInfo migratedAuthServerInfo;
     private final MigrationContext migrationContext = new MigrationContext();
@@ -72,17 +73,48 @@ public final class SuiteContext {
     }
 
     public ContainerInfo getAuthServerInfo() {
+        return getAuthServerInfo(0);
+    }
+
+    public ContainerInfo getAuthServerInfo(int dcIndex) {
+        return authServerInfo.get(dcIndex);
+    }
+
+    public List<ContainerInfo> getDcAuthServerInfo() {
         return authServerInfo;
     }
 
     public void setAuthServerInfo(ContainerInfo authServerInfo) {
-        this.authServerInfo = authServerInfo;
+        this.authServerInfo = new LinkedList<>();
+        this.authServerInfo.add(authServerInfo);
+    }
+
+    public void addAuthServerInfo(int dcIndex, ContainerInfo serverInfo) {
+        while (dcIndex >= authServerInfo.size()) {
+            authServerInfo.add(null);
+        }
+        this.authServerInfo.set(dcIndex, serverInfo);
     }
 
     public List<ContainerInfo> getAuthServerBackendsInfo() {
+        return getAuthServerBackendsInfo(0);
+    }
+
+    public List<ContainerInfo> getAuthServerBackendsInfo(int dcIndex) {
+        return authServerBackendsInfo.get(dcIndex);
+    }
+
+    public List<List<ContainerInfo>> getDcAuthServerBackendsInfo() {
         return authServerBackendsInfo;
     }
 
+    public void addAuthServerBackendsInfo(int dcIndex, ContainerInfo container) {
+        while (dcIndex >= authServerBackendsInfo.size()) {
+            authServerBackendsInfo.add(new LinkedList<>());
+        }
+        authServerBackendsInfo.get(dcIndex).add(container);
+    }
+
     public ContainerInfo getMigratedAuthServerInfo() {
         return migratedAuthServerInfo;
     }
@@ -96,7 +128,11 @@ public final class SuiteContext {
     }
 
     public boolean isAuthServerCluster() {
-        return !authServerBackendsInfo.isEmpty();
+        return ! authServerBackendsInfo.isEmpty();
+    }
+
+    public boolean isAuthServerCrossDc() {
+        return authServerBackendsInfo.size() > 1;
     }
 
     public boolean isAuthServerMigrationEnabled() {
@@ -113,19 +149,38 @@ public final class SuiteContext {
 
     @Override
     public String toString() {
-        String containers = "Auth server: " + (isAuthServerCluster() ? "\nFrontend: " : "")
+        StringBuilder sb = new StringBuilder("SUITE CONTEXT:\nAuth server: ");
-                + authServerInfo.getQualifier() + "\n";
+
-        for (ContainerInfo bInfo : getAuthServerBackendsInfo()) {
+        if (isAuthServerCrossDc()) {
-            containers += "Backend: " + bInfo + "\n";
+            for (int i = 0; i < authServerInfo.size(); i ++) {
+                ContainerInfo frontend = this.authServerInfo.get(i);
+                sb.append("\nFrontend (dc=").append(i).append("): ").append(frontend.getQualifier()).append("\n");
+            }
+
+            for (int i = 0; i < authServerBackendsInfo.size(); i ++) {
+                int dcIndex = i;
+                getDcAuthServerBackendsInfo().get(i).forEach(bInfo -> sb.append("Backend (dc=").append(dcIndex).append("): ").append(bInfo).append("\n"));
+            }
+        } else if (isAuthServerCluster()) {
+            sb.append(isAuthServerCluster() ? "\nFrontend: " : "")
+              .append(getAuthServerInfo().getQualifier())
+              .append("\n");
+
+            getAuthServerBackendsInfo().forEach(bInfo -> sb.append("  Backend: ").append(bInfo).append("\n"));
+        } else {
+          sb.append(getAuthServerInfo().getQualifier())
+            .append("\n");
         }
+
+
         if (isAuthServerMigrationEnabled()) {
-            containers += "Migrated from: " + System.getProperty("migrated.auth.server.version") + "\n";
+            sb.append("Migrated from: ").append(System.getProperty("migrated.auth.server.version")).append("\n");
         }
+
         if (isAdapterCompatTesting()) {
-            containers += "Adapter backward compatibility testing mode!\n";
+            sb.append("Adapter backward compatibility testing mode!\n");
         }
-        return "SUITE CONTEXT:\n"
+        return sb.toString();
-                + containers;
     }
 
 }

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/annotation/LoadBalancer.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.arquillian.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ *
+ * @author hmlnarik
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ ElementType.FIELD })
+public @interface LoadBalancer {
+    String value() default "";
+}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/containers/RegistryCreator.java

@@ -110,6 +110,8 @@ public class RegistryCreator {
             if (isClassPresent(getAdapterImplClassValue(containerDef))) {
                 return DeployableContainer.class.isAssignableFrom(
                         loadClass(getAdapterImplClassValue(containerDef)));
+            } else {
+                log.warn("Cannot load adapterImpl class for " + containerDef.getContainerName());
             }
         }
 

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/provider/LoadBalancerControllerProvider.java

@@ -0,0 +1,67 @@
+package org.keycloak.testsuite.arquillian.provider;
+
+import org.keycloak.testsuite.arquillian.annotation.LoadBalancer;
+import java.lang.annotation.Annotation;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import org.jboss.arquillian.container.spi.event.KillContainer;
+import org.jboss.arquillian.container.spi.event.StartContainer;
+import org.jboss.arquillian.container.spi.event.StopContainer;
+import org.jboss.arquillian.core.api.Instance;
+import org.jboss.arquillian.core.api.annotation.Inject;
+import org.jboss.arquillian.core.api.annotation.Observes;
+import org.jboss.arquillian.test.api.ArquillianResource;
+import org.jboss.arquillian.test.spi.enricher.resource.ResourceProvider;
+import org.keycloak.testsuite.arquillian.LoadBalancerController;
+import org.jboss.arquillian.container.spi.Container;
+import org.jboss.arquillian.container.spi.ContainerRegistry;
+
+/**
+ *
+ * @author hmlnarik
+ */
+public class LoadBalancerControllerProvider implements ResourceProvider {
+
+    @Inject
+    private Instance<ContainerRegistry> registry;
+
+    @Override
+    public boolean canProvide(Class<?> type) {
+        return type.equals(LoadBalancerController.class);
+    }
+
+    @Override
+    public Object lookup(ArquillianResource resource, Annotation... qualifiers) {
+        String balancerName = null;
+
+        // Check for the presence of possible qualifiers
+        for (Annotation a : qualifiers) {
+            Class<? extends Annotation> annotationType = a.annotationType();
+
+            if (annotationType.equals(LoadBalancer.class)) {
+                balancerName = ((LoadBalancer) a).value();
+            }
+        }
+
+        ContainerRegistry reg = registry.get();
+        Container container = null;
+        if (balancerName == null || "".equals(balancerName.trim())) {
+            if (reg.getContainers().size() == 1) {
+                container = reg.getContainers().get(0);
+            } else {
+                throw new IllegalArgumentException("Invalid load balancer configuration request - need to specify load balancer name in @LoadBalancerController");
+            }
+        } else {
+            container = reg.getContainer(balancerName);
+        }
+
+        if (container == null) {
+            throw new IllegalArgumentException("Invalid load balancer configuration - load balancer not found: '" + balancerName + "'");
+        }
+        if (! (container.getDeployableContainer() instanceof LoadBalancerController)) {
+            throw new IllegalArgumentException("Invalid load balancer configuration - container " + container.getName() + " is not a load balancer");
+        }
+
+        return container.getDeployableContainer();
+    }
+}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/SetSystemProperty.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.testsuite.util;
+
+import java.io.Closeable;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class SetSystemProperty implements Closeable {
+
+    private final String name;
+    private final String oldValue;
+
+    public SetSystemProperty(String name, String value) {
+        this.name = name;
+        this.oldValue = System.getProperty(name);
+
+        if (value == null) {
+            if (oldValue != null) {
+                System.getProperties().remove(name);
+            }
+        } else {
+            System.setProperty(name, value);
+        }
+    }
+
+    public void revert() {
+        String value = System.getProperty(name);
+
+        if (oldValue == null) {
+            if (value != null) {
+                System.getProperties().remove(name);
+            }
+        } else {
+            System.setProperty(name, oldValue);
+        }
+    }
+
+    @Override
+    public void close() {
+        revert();
+    }
+
+}

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractAdminCrossDCTest.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.crossdc;
+
+import org.keycloak.admin.client.resource.RealmResource;
+import org.keycloak.events.log.JBossLoggingEventListenerProviderFactory;
+import org.keycloak.representations.idm.RealmRepresentation;
+import org.keycloak.testsuite.events.EventsListenerProviderFactory;
+import org.keycloak.testsuite.util.TestCleanup;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.junit.Before;
+
+/**
+ *
+ * @author hmlnarik
+ */
+public abstract class AbstractAdminCrossDCTest extends AbstractCrossDCTest {
+
+    protected static final String REALM_NAME = "admin-client-test";
+
+    protected RealmResource realm;
+    protected String realmId;
+
+
+    @Override
+    public void configureTestRealm(RealmRepresentation testRealm) {
+        findTestApp(testRealm).setDirectAccessGrantsEnabled(true);
+    }
+
+
+
+    @Override
+    public void addTestRealms(List<RealmRepresentation> testRealms) {
+        super.addTestRealms(testRealms);
+
+        RealmRepresentation adminRealmRep = new RealmRepresentation();
+        adminRealmRep.setId(REALM_NAME);
+        adminRealmRep.setRealm(REALM_NAME);
+        adminRealmRep.setEnabled(true);
+        Map<String, String> config = new HashMap<>();
+        config.put("from", "auto@keycloak.org");
+        config.put("host", "localhost");
+        config.put("port", "3025");
+        adminRealmRep.setSmtpServer(config);
+
+        List<String> eventListeners = new ArrayList<>();
+        eventListeners.add(JBossLoggingEventListenerProviderFactory.ID);
+        eventListeners.add(EventsListenerProviderFactory.PROVIDER_ID);
+        adminRealmRep.setEventsListeners(eventListeners);
+
+        testRealms.add(adminRealmRep);
+    }
+
+    @Before
+    public void setRealm() {
+        realm = adminClient.realm(REALM_NAME);
+        realmId = realm.toRepresentation().getId();
+    }
+
+    @Override
+    protected TestCleanup getCleanup() {
+        return getCleanup(REALM_NAME);
+    }
+}

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractCrossDCTest.java

@@ -0,0 +1,161 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.crossdc;
+
+import org.keycloak.admin.client.Keycloak;
+import org.keycloak.models.Constants;
+import org.keycloak.testsuite.AbstractTestRealmKeycloakTest;
+import org.keycloak.testsuite.arquillian.ContainerInfo;
+import org.keycloak.testsuite.arquillian.LoadBalancerController;
+import org.keycloak.testsuite.arquillian.annotation.LoadBalancer;
+import org.keycloak.testsuite.auth.page.AuthRealm;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
+import org.jboss.arquillian.container.test.api.ContainerController;
+import org.jboss.arquillian.test.api.ArquillianResource;
+import org.junit.After;
+import org.junit.Before;
+
+/**
+ *
+ * @author hmlnarik
+ */
+public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest {
+
+    @ArquillianResource
+    @LoadBalancer(value = "auth-server-balancer-cross-dc")
+    protected LoadBalancerController loadBalancerCtrl;
+
+    @ArquillianResource
+    protected ContainerController containerController;
+
+    protected Map<ContainerInfo, Keycloak> backendAdminClients = new HashMap<>();
+
+    @After
+    @Before
+    public void enableOnlyFirstNodeInFirstDc() {
+        this.loadBalancerCtrl.disableAllBackendNodes();
+        loadBalancerCtrl.enableBackendNodeByName(getAutomaticallyStartedBackendNodes(0)
+          .findFirst()
+          .orElseThrow(() -> new IllegalStateException("No node is started automatically"))
+          .getQualifier()
+        );
+    }
+
+    @Before
+    public void terminateManuallyStartedServers() {
+        log.debug("Halting all nodes that are started manually");
+        this.suiteContext.getDcAuthServerBackendsInfo().stream()
+          .flatMap(List::stream)
+          .filter(ContainerInfo::isStarted)
+          .filter(ContainerInfo::isManual)
+          .map(ContainerInfo::getQualifier)
+          .forEach(containerController::stop);
+    }
+
+    @Override
+    public void importTestRealms() {
+        enableOnlyFirstNodeInFirstDc();
+        super.importTestRealms();
+    }
+
+    @Override
+    public void afterAbstractKeycloakTest() {
+        enableOnlyFirstNodeInFirstDc();
+        super.afterAbstractKeycloakTest();
+    }
+
+    @Override
+    public void deleteCookies() {
+        enableOnlyFirstNodeInFirstDc();
+        super.deleteCookies();
+    }
+
+    @Before
+    public void initLoadBalancer() {
+        log.debug("Initializing load balancer - only enabling started nodes in the first DC");
+        this.loadBalancerCtrl.disableAllBackendNodes();
+        // Enable only the started nodes in each datacenter
+        this.suiteContext.getDcAuthServerBackendsInfo().get(0).stream()
+          .filter(ContainerInfo::isStarted)
+          .map(ContainerInfo::getQualifier)
+          .forEach(loadBalancerCtrl::enableBackendNodeByName);
+    }
+
+    protected Keycloak createAdminClientFor(ContainerInfo node) {
+        log.info("Initializing admin client for " + node.getContextRoot() + "/auth");
+        return Keycloak.getInstance(node.getContextRoot() + "/auth", AuthRealm.MASTER, AuthRealm.ADMIN, AuthRealm.ADMIN, Constants.ADMIN_CLI_CLIENT_ID);
+    }
+
+    protected Keycloak getAdminClientFor(ContainerInfo node) {
+        Keycloak adminClient = backendAdminClients.get(node);
+        if (adminClient == null && node.equals(suiteContext.getAuthServerInfo())) {
+            adminClient = this.adminClient;
+        }
+        return adminClient;
+    }
+
+    public void disableDcOnLoadBalancer(int dcIndex) {
+        log.infof("Disabling load balancer for dc=%d", dcIndex);
+        this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex).forEach(c -> loadBalancerCtrl.disableBackendNodeByName(c.getQualifier()));
+    }
+
+    /**
+     * Enables all started nodes in the given data center
+     * @param dcIndex
+     */
+    public void enableDcOnLoadBalancer(int dcIndex) {
+        log.infof("Enabling load balancer for dc=%d", dcIndex);
+        final List<ContainerInfo> dcNodes = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex);
+        if (! dcNodes.stream().anyMatch(ContainerInfo::isStarted)) {
+            log.warnf("No node is started in DC %d", dcIndex);
+        } else {
+            dcNodes.stream()
+              .filter(ContainerInfo::isStarted)
+              .forEach(c -> loadBalancerCtrl.enableBackendNodeByName(c.getQualifier()));
+        }
+    }
+
+    public void disableLoadBalancerNode(int dcIndex, int nodeIndex) {
+        log.infof("Disabling load balancer for dc=%d, node=%d", dcIndex, nodeIndex);
+        loadBalancerCtrl.disableBackendNodeByName(this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex).get(nodeIndex).getQualifier());
+    }
+
+    public void enableLoadBalancerNode(int dcIndex, int nodeIndex) {
+        log.infof("Enabling load balancer for dc=%d, node=%d", dcIndex, nodeIndex);
+        final ContainerInfo backendNode = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex).get(nodeIndex);
+        if (backendNode == null) {
+            throw new IllegalArgumentException("Invalid node with index " + nodeIndex + " for DC " + dcIndex);
+        }
+        if (! backendNode.isStarted()) {
+            log.warnf("Node %s is not started in DC %d", backendNode.getQualifier(), dcIndex);
+        }
+        loadBalancerCtrl.enableBackendNodeByName(backendNode.getQualifier());
+    }
+
+    public Stream<ContainerInfo> getManuallyStartedBackendNodes(int dcIndex) {
+        final List<ContainerInfo> dcNodes = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex);
+        return dcNodes.stream().filter(ContainerInfo::isManual);
+    }
+
+    public Stream<ContainerInfo> getAutomaticallyStartedBackendNodes(int dcIndex) {
+        final List<ContainerInfo> dcNodes = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex);
+        return dcNodes.stream().filter(c -> ! c.isManual());
+    }
+}

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/ActionTokenCrossDCTest.java

@@ -0,0 +1,152 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.crossdc;
+
+import org.keycloak.admin.client.resource.UserResource;
+import org.keycloak.events.admin.OperationType;
+import org.keycloak.events.admin.ResourceType;
+import org.keycloak.models.UserModel;
+import org.keycloak.representations.idm.UserRepresentation;
+import org.keycloak.testsuite.Retry;
+import org.keycloak.testsuite.admin.ApiUtil;
+import org.keycloak.testsuite.arquillian.ContainerInfo;
+import org.keycloak.testsuite.page.LoginPasswordUpdatePage;
+import org.keycloak.testsuite.pages.ErrorPage;
+import org.keycloak.testsuite.util.AdminEventPaths;
+import org.keycloak.testsuite.util.GreenMailRule;
+import org.keycloak.testsuite.util.MailUtils;
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+import javax.mail.MessagingException;
+import javax.mail.internet.MimeMessage;
+import javax.ws.rs.core.Response;
+import org.jboss.arquillian.graphene.page.Page;
+import org.jboss.arquillian.test.api.ArquillianResource;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+
+/**
+ *
+ * @author hmlnarik
+ */
+public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest {
+
+    @Rule
+    public GreenMailRule greenMail = new GreenMailRule();
+
+    @Page
+    protected LoginPasswordUpdatePage passwordUpdatePage;
+
+    @Page
+    protected ErrorPage errorPage;
+
+    private String createUser(UserRepresentation userRep) {
+        Response response = realm.users().create(userRep);
+        String createdId = ApiUtil.getCreatedId(response);
+        response.close();
+
+        getCleanup().addUserId(createdId);
+
+        return createdId;
+    }
+
+    @Test
+    public void sendResetPasswordEmailSuccessWorksInCrossDc() throws IOException, MessagingException {
+        UserRepresentation userRep = new UserRepresentation();
+        userRep.setEnabled(true);
+        userRep.setUsername("user1");
+        userRep.setEmail("user1@test.com");
+
+        String id = createUser(userRep);
+
+        UserResource user = realm.users().get(id);
+        List<String> actions = new LinkedList<>();
+        actions.add(UserModel.RequiredAction.UPDATE_PASSWORD.name());
+        user.executeActionsEmail(actions);
+
+        Assert.assertEquals(1, greenMail.getReceivedMessages().length);
+
+        MimeMessage message = greenMail.getReceivedMessages()[0];
+
+        String link = MailUtils.getPasswordResetEmailLink(message);
+
+        driver.navigate().to(link);
+
+        passwordUpdatePage.assertCurrent();
+
+        passwordUpdatePage.changePassword("new-pass", "new-pass");
+
+        assertEquals("Your account has been updated.", driver.getTitle());
+
+        disableDcOnLoadBalancer(0);
+        enableDcOnLoadBalancer(1);
+
+        Retry.execute(() -> {
+            driver.navigate().to(link);
+            errorPage.assertCurrent();
+        }, 3, 400);
+    }
+
+    @Ignore("KEYCLOAK-5030")
+    @Test
+    public void sendResetPasswordEmailAfterNewNodeAdded() throws IOException, MessagingException {
+        disableDcOnLoadBalancer(1);
+
+        UserRepresentation userRep = new UserRepresentation();
+        userRep.setEnabled(true);
+        userRep.setUsername("user1");
+        userRep.setEmail("user1@test.com");
+
+        String id = createUser(userRep);
+
+        UserResource user = realm.users().get(id);
+        List<String> actions = new LinkedList<>();
+        actions.add(UserModel.RequiredAction.UPDATE_PASSWORD.name());
+        user.executeActionsEmail(actions);
+
+        Assert.assertEquals(1, greenMail.getReceivedMessages().length);
+
+        MimeMessage message = greenMail.getReceivedMessages()[0];
+
+        String link = MailUtils.getPasswordResetEmailLink(message);
+
+        driver.navigate().to(link);
+
+        passwordUpdatePage.assertCurrent();
+
+        passwordUpdatePage.changePassword("new-pass", "new-pass");
+
+        assertEquals("Your account has been updated.", driver.getTitle());
+
+        disableDcOnLoadBalancer(0);
+        getManuallyStartedBackendNodes(1)
+          .findFirst()
+          .ifPresent(c -> {
+              containerController.start(c.getQualifier());
+              loadBalancerCtrl.enableBackendNodeByName(c.getQualifier());
+          });
+
+        driver.navigate().to(link);
+
+        errorPage.assertCurrent();
+    }
+
+}

testsuite/integration-arquillian/tests/base/src/test/resources/META-INF/keycloak-server.json

@@ -107,15 +107,23 @@
 
     "connectionsInfinispan": {
         "default": {
+            "jgroupsUdpMcastAddr": "${keycloak.connectionsInfinispan.jgroupsUdpMcastAddr:234.56.78.90}",
+            "nodeName": "${keycloak.connectionsInfinispan.nodeName,jboss.node.name:defaultNodeName}",
             "clustered": "${keycloak.connectionsInfinispan.clustered:false}",
             "async": "${keycloak.connectionsInfinispan.async:false}",
             "sessionsOwners": "${keycloak.connectionsInfinispan.sessionsOwners:1}",
             "l1Lifespan": "${keycloak.connectionsInfinispan.l1Lifespan:600000}",
             "remoteStoreEnabled": "${keycloak.connectionsInfinispan.remoteStoreEnabled:false}",
-            "remoteStoreHost": "${keycloak.connectionsInfinispan.remoteStoreHost:localhost}",
+            "remoteStoreServer": "${keycloak.connectionsInfinispan.remoteStoreServer:localhost}",
             "remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort:11222}"
         }
     },
+    
+    "stickySessionEncoder": {
+        "infinispan": {
+            "nodeName": "${keycloak.stickySessionEncoder.nodeName,jboss.node.name:defaultNodeName}"
+        }
+    },
 
 
     "truststore": {

testsuite/integration-arquillian/tests/base/src/test/resources/arquillian.xml

@@ -103,6 +103,7 @@
                 <property name="outputToConsole">${backends.console.output}</property>
                 <property name="managementPort">${auth.server.backend1.management.port}</property>
                 <property name="startupTimeoutInSeconds">${auth.server.jboss.startup.timeout}</property>
+                <property name="bindHttpPortOffset">${auth.server.backend1.port.offset}</property>
             </configuration>
         </container>
         <container qualifier="auth-server-${auth.server}-backend2" mode="manual" >
@@ -124,6 +125,7 @@
                 <property name="outputToConsole">${backends.console.output}</property>
                 <property name="managementPort">${auth.server.backend2.management.port}</property>
                 <property name="startupTimeoutInSeconds">${auth.server.jboss.startup.timeout}</property>
+                <property name="bindHttpPortOffset">${auth.server.backend2.port.offset}</property>
             </configuration>
         </container>
     </group>
@@ -165,6 +167,138 @@
     </group>
 
 
+    <!-- Cross DC with embedded undertow. Node numbering is [centre #].[node #] -->
+    <group qualifier="auth-server-undertow-cross-dc">
+        <container qualifier="cache-server-cross-dc" mode="suite" >
+            <configuration>
+                <property name="enabled">${auth.server.undertow.crossdc}</property>
+                <property name="adapterImplClass">org.jboss.as.arquillian.container.managed.ManagedDeployableContainer</property>
+                <property name="jbossHome">${cache.server.home}</property>
+                <property name="serverConfig">standalone.xml</property>
+                <property name="jbossArguments">
+                    -Djboss.socket.binding.port-offset=${cache.server.port.offset}
+                    -Djboss.default.multicast.address=234.56.78.99
+                    -Djboss.node.name=cache-server
+                    ${adapter.test.props}
+                    ${auth.server.profile}
+                </property>
+                <property name="javaVmArguments">
+                    ${auth.server.memory.settings}
+                    -Djava.net.preferIPv4Stack=true
+                </property>
+                <property name="outputToConsole">${cache.server.console.output}</property>
+                <property name="managementPort">${cache.server.management.port}</property>
+                <property name="startupTimeoutInSeconds">${auth.server.jboss.startup.timeout}</property>
+            </configuration>
+        </container>
+
+        <container qualifier="auth-server-balancer-cross-dc" mode="suite" >
+            <configuration>
+                <property name="enabled">${auth.server.undertow.crossdc}</property>
+                <property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.lb.SimpleUndertowLoadBalancerContainer</property>
+                <property name="bindAddress">localhost</property>
+                <property name="bindHttpPort">${auth.server.http.port}</property>
+                <property name="bindHttpPortOffset">5</property>
+                <property name="nodes">auth-server-undertow-cross-dc-0.1=http://localhost:8101,auth-server-undertow-cross-dc-0.2-manual=http://localhost:8102,auth-server-undertow-cross-dc-1.1=http://localhost:8111,auth-server-undertow-cross-dc-1.2-manual=http://localhost:8112</property>
+            </configuration>
+        </container>
+
+        <container qualifier="auth-server-undertow-cross-dc-0.1" mode="suite" >
+            <configuration>
+                <property name="enabled">${auth.server.undertow.crossdc}</property>
+                <property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.KeycloakOnUndertow</property>
+                <property name="bindAddress">localhost</property>
+                <property name="bindHttpPort">${auth.server.http.port}</property>
+                <property name="bindHttpPortOffset">-79</property>
+                <property name="route">auth-server-undertow-cross-dc-0.1</property>
+                <property name="remoteMode">${undertow.remote}</property>
+                <property name="dataCenter">0</property>
+                <property name="keycloakConfigPropertyOverrides">{
+                    "keycloak.connectionsInfinispan.jgroupsUdpMcastAddr": "234.56.78.1",
+                    "keycloak.connectionsInfinispan.nodeName": "auth-server-undertow-cross-dc-0.1",
+                    "keycloak.connectionsInfinispan.clustered": "${keycloak.connectionsInfinispan.clustered:true}",
+                    "keycloak.connectionsInfinispan.remoteStoreServer": "${keycloak.connectionsInfinispan.remoteStoreServer:localhost}",
+                    "keycloak.connectionsInfinispan.remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort:11222}",
+                    "keycloak.connectionsInfinispan.remoteStoreEnabled": "${keycloak.connectionsInfinispan.remoteStoreEnabled:true}",
+                    "keycloak.connectionsJpa.url": "${keycloak.connectionsJpa.url.crossdc:jdbc:h2:mem:test-dc-shared}",
+                    "keycloak.connectionsJpa.driver": "${keycloak.connectionsJpa.driver.crossdc:org.h2.Driver}",
+                    "keycloak.connectionsJpa.driverDialect": "${keycloak.connectionsJpa.driverDialect.crossdc:}"
+                }</property>
+            </configuration>
+        </container>
+        <container qualifier="auth-server-undertow-cross-dc-0.2-manual" mode="manual" >
+            <configuration>
+                <property name="enabled">${auth.server.undertow.crossdc}</property>
+                <property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.KeycloakOnUndertow</property>
+                <property name="bindAddress">localhost</property>
+                <property name="bindHttpPort">${auth.server.http.port}</property>
+                <property name="bindHttpPortOffset">-78</property>
+                <property name="route">auth-server-undertow-cross-dc-0.2</property>
+                <property name="remoteMode">${undertow.remote}</property>
+                <property name="dataCenter">0</property>
+                <property name="keycloakConfigPropertyOverrides">{
+                    "keycloak.connectionsInfinispan.jgroupsUdpMcastAddr": "234.56.78.1",
+                    "keycloak.connectionsInfinispan.nodeName": "auth-server-undertow-cross-dc-0.2",
+                    "keycloak.connectionsInfinispan.clustered": "${keycloak.connectionsInfinispan.clustered:true}",
+                    "keycloak.connectionsInfinispan.remoteStoreServer": "${keycloak.connectionsInfinispan.remoteStoreServer:localhost}",
+                    "keycloak.connectionsInfinispan.remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort:11222}",
+                    "keycloak.connectionsInfinispan.remoteStoreEnabled": "${keycloak.connectionsInfinispan.remoteStoreEnabled:true}",
+                    "keycloak.connectionsJpa.url": "${keycloak.connectionsJpa.url.crossdc:jdbc:h2:mem:test-dc-shared}",
+                    "keycloak.connectionsJpa.driver": "${keycloak.connectionsJpa.driver.crossdc:org.h2.Driver}",
+                    "keycloak.connectionsJpa.driverDialect": "${keycloak.connectionsJpa.driverDialect.crossdc:}"
+                }</property>
+            </configuration>
+        </container>
+
+        <container qualifier="auth-server-undertow-cross-dc-1.1" mode="suite" >
+            <configuration>
+                <property name="enabled">${auth.server.undertow.crossdc}</property>
+                <property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.KeycloakOnUndertow</property>
+                <property name="bindAddress">localhost</property>
+                <property name="bindHttpPort">${auth.server.http.port}</property>
+                <property name="bindHttpPortOffset">-69</property>
+                <property name="route">auth-server-undertow-cross-dc-1.1</property>
+                <property name="remoteMode">${undertow.remote}</property>
+                <property name="dataCenter">1</property>
+                <property name="keycloakConfigPropertyOverrides">{
+                    "keycloak.connectionsInfinispan.jgroupsUdpMcastAddr": "234.56.78.2",
+                    "keycloak.connectionsInfinispan.nodeName": "auth-server-undertow-cross-dc-1.1",
+                    "keycloak.connectionsInfinispan.clustered": "${keycloak.connectionsInfinispan.clustered:true}",
+                    "keycloak.connectionsInfinispan.remoteStoreServer": "${keycloak.connectionsInfinispan.remoteStoreServer:localhost}",
+                    "keycloak.connectionsInfinispan.remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort:11222}",
+                    "keycloak.connectionsInfinispan.remoteStoreEnabled": "${keycloak.connectionsInfinispan.remoteStoreEnabled:true}",
+                    "keycloak.connectionsJpa.url": "${keycloak.connectionsJpa.url.crossdc:jdbc:h2:mem:test-dc-shared}",
+                    "keycloak.connectionsJpa.driver": "${keycloak.connectionsJpa.driver.crossdc:org.h2.Driver}",
+                    "keycloak.connectionsJpa.driverDialect": "${keycloak.connectionsJpa.driverDialect.crossdc:}"
+                }</property>
+            </configuration>
+        </container>
+        <container qualifier="auth-server-undertow-cross-dc-1.2-manual" mode="manual" >
+            <configuration>
+                <property name="enabled">${auth.server.undertow.crossdc}</property>
+                <property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.KeycloakOnUndertow</property>
+                <property name="bindAddress">localhost</property>
+                <property name="bindHttpPort">${auth.server.http.port}</property>
+                <property name="bindHttpPortOffset">-68</property>
+                <property name="route">auth-server-undertow-cross-dc-1.2</property>
+                <property name="remoteMode">${undertow.remote}</property>
+                <property name="dataCenter">1</property>
+                <property name="keycloakConfigPropertyOverrides">{
+                    "keycloak.connectionsInfinispan.jgroupsUdpMcastAddr": "234.56.78.2",
+                    "keycloak.connectionsInfinispan.nodeName": "auth-server-undertow-cross-dc-1.2",
+                    "keycloak.connectionsInfinispan.clustered": "${keycloak.connectionsInfinispan.clustered:true}",
+                    "keycloak.connectionsInfinispan.remoteStoreServer": "${keycloak.connectionsInfinispan.remoteStoreServer:localhost}",
+                    "keycloak.connectionsInfinispan.remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort:11222}",
+                    "keycloak.connectionsInfinispan.remoteStoreEnabled": "${keycloak.connectionsInfinispan.remoteStoreEnabled:true}",
+                    "keycloak.connectionsJpa.url": "${keycloak.connectionsJpa.url.crossdc:jdbc:h2:mem:test-dc-shared}",
+                    "keycloak.connectionsJpa.driver": "${keycloak.connectionsJpa.driver.crossdc:org.h2.Driver}",
+                    "keycloak.connectionsJpa.driverDialect": "${keycloak.connectionsJpa.driverDialect.crossdc:}"
+                }</property>
+            </configuration>
+        </container>
+    </group>
+
+
     <container qualifier="auth-server-balancer-wildfly" mode="suite" >
         <configuration>
             <property name="enabled">${auth.server.cluster}</property>

testsuite/integration-arquillian/tests/pom.xml

@@ -41,6 +41,7 @@
     <properties>
         <auth.server>undertow</auth.server>
         <auth.server.undertow>true</auth.server.undertow>
+        <auth.server.undertow.crossdc>true</auth.server.undertow.crossdc>
 
         <auth.server.container>auth-server-${auth.server}</auth.server.container>
         <auth.server.home>${containers.home}/${auth.server.container}</auth.server.home>
@@ -65,7 +66,17 @@
         <auth.server.remote>false</auth.server.remote>
         <auth.server.profile/>
         <auth.server.feature/>
-    
+
+        <cache.server>infinispan</cache.server>
+        <cache.server.container>cache-server-${cache.server}</cache.server.container>
+        <cache.server.home>${containers.home}/${cache.server.container}</cache.server.home>
+        <cache.server.port.offset>1010</cache.server.port.offset>
+        <cache.server.management.port>11000</cache.server.management.port>
+        <cache.server.console.output>true</cache.server.console.output>
+        <keycloak.connectionsInfinispan.remoteStoreServer>localhost</keycloak.connectionsInfinispan.remoteStoreServer>
+        <keycloak.connectionsInfinispan.remoteStorePort>12232</keycloak.connectionsInfinispan.remoteStorePort>
+        <keycloak.connectionsJpa.url.crossdc>jdbc:h2:mem:test-dc-shared</keycloak.connectionsJpa.url.crossdc>
+
         <adapter.test.props/>
         <migration.import.properties/>
         <examples.home>${project.build.directory}/examples</examples.home>
@@ -289,6 +300,7 @@
                 <auth.server>wildfly</auth.server>
                 <auth.server.jboss>true</auth.server.jboss>
                 <auth.server.undertow>false</auth.server.undertow>
+                <auth.server.undertow.crossdc>false</auth.server.undertow.crossdc>
                 <auth.server.config.property.value>standalone.xml</auth.server.config.property.value>
                 <auth.server.config.dir>${auth.server.home}/standalone/configuration</auth.server.config.dir>
                 <h2.version>1.3.173</h2.version>
@@ -307,6 +319,7 @@
                 <auth.server>eap</auth.server>
                 <auth.server.jboss>true</auth.server.jboss>
                 <auth.server.undertow>false</auth.server.undertow>
+                <auth.server.undertow.crossdc>false</auth.server.undertow.crossdc>
                 <auth.server.config.property.value>standalone.xml</auth.server.config.property.value>
                 <auth.server.config.dir>${auth.server.home}/standalone/configuration</auth.server.config.dir>
                 <h2.version>1.3.173</h2.version>
@@ -319,6 +332,110 @@
             </dependencies>
         </profile>
 
+        <profile>
+            <id>cache-server-infinispan</id>
+            <properties>
+                <cache.server>infinispan</cache.server>
+                <cache.server.jboss>true</cache.server.jboss>
+                <cache.server.config.dir>${cache.server.home}/standalone/configuration</cache.server.config.dir>                
+            </properties>
+            <dependencies>
+                <dependency>
+                    <groupId>org.wildfly.arquillian</groupId>
+                    <artifactId>wildfly-arquillian-container-managed</artifactId>
+                </dependency>
+<!--                <dependency>
+                    <groupId>org.infinispan.arquillian.container</groupId>
+                    <artifactId>infinispan-arquillian-impl</artifactId>
+                    <scope>test</scope>
+                    <exclusions>
+                        <exclusion>
+                            <groupId>org.infinispan</groupId>
+                            <artifactId>infinispan-core</artifactId>
+                        </exclusion>
+                    </exclusions>
+                </dependency>-->
+            </dependencies>
+
+            <build>
+                <plugins>
+                    <plugin>
+                        <artifactId>maven-enforcer-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <goals>
+                                    <goal>enforce</goal>
+                                </goals>
+                                <configuration>
+                                    <rules>
+                                        <!--requireActiveProfile 'auth-server-wildfly/eap' doesn't work unless the profiles are defined in all submodule poms
+                                        using requireProperty instead-->
+                                        <requireProperty>
+                                            <property>cache.server</property>
+                                            <regex>(infinispan)|(jdg)</regex>
+                                            <regexMessage>Profile "cache-server-infinispan" requires activation of profile "cache-server-infinispan" or "cache-server-jdg".</regexMessage>
+                                        </requireProperty>
+                                    </rules>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                    <plugin>
+                        <artifactId>maven-antrun-plugin</artifactId>
+                    </plugin>
+                </plugins>
+                <pluginManagement>
+                    <plugins>
+                        <plugin>
+                            <artifactId>maven-dependency-plugin</artifactId>
+                            <executions>
+                                <execution>
+                                    <id>unpack-cache-server-infinispan</id>
+                                    <phase>generate-resources</phase>
+                                    <goals>
+                                        <goal>unpack</goal>
+                                    </goals>
+                                    <configuration>
+                                        <artifactItems>
+                                            <artifactItem>
+                                                <groupId>org.keycloak.testsuite</groupId>
+                                                <artifactId>integration-arquillian-servers-cache-server-infinispan</artifactId>
+                                                <version>${project.version}</version>
+                                                <type>zip</type>
+                                                <outputDirectory>${containers.home}</outputDirectory>
+                                            </artifactItem>
+                                        </artifactItems>
+                                        <overWriteIfNewer>true</overWriteIfNewer>
+                                    </configuration>
+                                </execution>
+                            </executions>
+                        </plugin>
+                        <plugin>
+                            <artifactId>maven-surefire-plugin</artifactId>
+                            <configuration>
+                                <systemPropertyVariables>
+                                    <auth.server.crossdc>true</auth.server.crossdc>
+                                    <auth.server.undertow.crossdc>${auth.server.undertow.crossdc}</auth.server.undertow.crossdc>
+
+                                    <cache.server>${cache.server}</cache.server>
+                                    <cache.server.port.offset>${cache.server.port.offset}</cache.server.port.offset>
+                                    <cache.server.container>${cache.server.container}</cache.server.container>
+                                    <cache.server.home>${cache.server.home}</cache.server.home>
+                                    <cache.server.console.output>${cache.server.console.output}</cache.server.console.output>
+                                    <cache.server.management.port>${cache.server.management.port}</cache.server.management.port>
+                                    
+                                    <keycloak.connectionsInfinispan.remoteStorePort>${keycloak.connectionsInfinispan.remoteStorePort}</keycloak.connectionsInfinispan.remoteStorePort>
+                                    <keycloak.connectionsInfinispan.remoteStoreServer>${keycloak.connectionsInfinispan.remoteStoreServer}</keycloak.connectionsInfinispan.remoteStoreServer>
+                                    
+                                    <keycloak.connectionsJpa.url.crossdc>${keycloak.connectionsJpa.url.crossdc}</keycloak.connectionsJpa.url.crossdc>
+                                </systemPropertyVariables>
+                            </configuration>
+                        </plugin>
+                    </plugins>
+                </pluginManagement>
+            </build>
+        </profile>
+
         <profile>
             <id>auth-server-profile</id>
             <activation>