Merge pull request #4240 from hmlnarik/KEYCLOAK-4189-Cross-DC-testing

KEYCLOAK-4189 Infinispan cache and channel statistics for Cross-DC testing

model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java

@@ -156,8 +156,12 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
             String nodeName = config.get("nodeName", System.getProperty(InfinispanConnectionProvider.JBOSS_NODE_NAME));
             String jgroupsUdpMcastAddr = config.get("jgroupsUdpMcastAddr", System.getProperty(InfinispanConnectionProvider.JGROUPS_UDP_MCAST_ADDR));
             configureTransport(gcb, nodeName, jgroupsUdpMcastAddr);
+            gcb.globalJmxStatistics()
+              .jmxDomain(InfinispanConnectionProvider.JMX_DOMAIN + "-" + nodeName);
         }
-        gcb.globalJmxStatistics().allowDuplicateDomains(allowDuplicateJMXDomains);
+        gcb.globalJmxStatistics()
+          .allowDuplicateDomains(allowDuplicateJMXDomains)
+          .enable();
 
         cacheManager = new DefaultCacheManager(gcb.build());
         containerManaged = false;
@@ -339,8 +343,13 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
                     channel.setName(nodeName);
                     JGroupsTransport transport = new JGroupsTransport(channel);
 
-                    gcb.transport().nodeName(nodeName);
+                    gcb.transport()
-                    gcb.transport().transport(transport);
+                      .nodeName(nodeName)
+                      .transport(transport)
+                      .globalJmxStatistics()
+                        .jmxDomain(InfinispanConnectionProvider.JMX_DOMAIN + "-" + nodeName)
+                        .enable()
+                      ;
 
                     logger.infof("Configured jgroups transport with the channel name: %s", nodeName);
                 } catch (Exception e) {

model/infinispan/src/main/java/org/keycloak/connections/infinispan/InfinispanConnectionProvider.java

@@ -55,6 +55,7 @@ public interface InfinispanConnectionProvider extends Provider {
     String JBOSS_NODE_NAME = "jboss.node.name";
     String JGROUPS_UDP_MCAST_ADDR = "jgroups.udp.mcast_addr";
 
+    String JMX_DOMAIN = "jboss.datagrid-infinispan";
 
     <K, V> Cache<K, V> getCache(String name);
 

testsuite/integration-arquillian/servers/cache-server/jboss/common/add-keycloak-caches.xsl

@@ -33,6 +33,14 @@
         </xsl:copy>
     </xsl:template>
 
+    <xsl:template match="//*[local-name()='subsystem' and starts-with(namespace-uri(), $nsCacheServer)]
+                        /*[local-name()='cache-container' and starts-with(namespace-uri(), $nsCacheServer) and @name='clustered']">
+        <xsl:copy>
+            <xsl:apply-templates select="@* | node()" />
+            <replicated-cache name="work" start="EAGER" batching="false" />
+        </xsl:copy>
+    </xsl:template>
+
     <xsl:template match="@*|node()">
         <xsl:copy>
             <xsl:apply-templates select="@*|node()" />

testsuite/integration-arquillian/servers/cache-server/jboss/pom.xml

@@ -100,7 +100,7 @@
                         <artifactId>xml-maven-plugin</artifactId>
                         <executions>
                             <execution>
-                                <id>configure-adapter-debug-log</id>
+                                <id>configure-keycloak-caches</id>
                                 <phase>process-test-resources</phase>
                                 <goals>
                                     <goal>transform</goal>
@@ -111,8 +111,9 @@
                                             <dir>${cache.server.jboss.home}/standalone/configuration</dir>
                                             <includes>
                                                 <include>standalone.xml</include>
+                                                <include>clustered.xml</include>
                                             </includes>
-                                            <stylesheet>${common.resources}/add-keycloak-remote-store.xsl</stylesheet>
+                                            <stylesheet>${common.resources}/add-keycloak-caches.xsl</stylesheet>
                                             <outputDir>${cache.server.jboss.home}/standalone/configuration</outputDir>
                                         </transformationSet>
                                     </transformationSets>
@@ -173,6 +174,23 @@
                                     <overwrite>true</overwrite>
                                 </configuration>
                             </execution>
+                            <execution>
+                                <id>copy-cache-server-configuration-for-dc-2</id>
+                                <phase>process-resources</phase>
+                                <goals>
+                                    <goal>copy-resources</goal>
+                                </goals>
+                                <configuration>
+                                    <outputDirectory>${cache.server.jboss.home}/standalone-dc-2/deployments</outputDirectory>
+                                    <includeEmptyDirs>true</includeEmptyDirs>
+                                    <resources>
+                                        <resource>
+                                            <directory>${cache.server.jboss.home}/standalone/deployments</directory>
+                                        </resource>
+                                    </resources>
+                                    <overwrite>true</overwrite>
+                                </configuration>
+                            </execution>
                         </executions>
                     </plugin>
 

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/AuthServerTestEnricher.java

@@ -142,6 +142,7 @@ public class AuthServerTestEnricher {
 
             containers.stream()
               .filter(c -> c.getQualifier().startsWith(AUTH_SERVER_CONTAINER + "-cross-dc-"))
+              .sorted((a, b) -> a.getQualifier().compareTo(b.getQualifier()))
               .forEach(c -> {
                 String portOffsetString = c.getArquillianContainer().getContainerConfiguration().getContainerProperties().getOrDefault("bindHttpPortOffset", "0");
                 String dcString = c.getArquillianContainer().getContainerConfiguration().getContainerProperties().getOrDefault("dataCenter", "0");

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/CacheStatisticsControllerEnricher.java

@@ -0,0 +1,356 @@
+package org.keycloak.testsuite.arquillian;
+
+import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
+import org.keycloak.testsuite.Retry;
+import java.util.Map;
+import org.jboss.arquillian.core.api.Instance;
+import org.jboss.arquillian.core.api.annotation.Inject;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.net.MalformedURLException;
+import javax.management.MBeanServerConnection;
+import javax.management.ObjectName;
+import javax.management.remote.JMXConnector;
+import javax.management.remote.JMXServiceURL;
+import org.jboss.arquillian.container.spi.Container;
+import org.jboss.arquillian.container.spi.ContainerRegistry;
+import org.jboss.arquillian.test.spi.TestEnricher;
+import java.io.IOException;
+import java.lang.reflect.Parameter;
+import java.util.Arrays;
+import java.util.stream.Collectors;
+import javax.management.Attribute;
+import javax.management.AttributeNotFoundException;
+import javax.management.InstanceNotFoundException;
+import javax.management.IntrospectionException;
+import javax.management.MBeanAttributeInfo;
+import javax.management.MBeanException;
+import javax.management.MBeanInfo;
+import javax.management.MalformedObjectNameException;
+import javax.management.ReflectionException;
+import org.keycloak.testsuite.arquillian.annotation.JmxInfinispanCacheStatistics;
+import java.util.Set;
+import org.keycloak.testsuite.arquillian.annotation.JmxInfinispanChannelStatistics;
+import org.keycloak.testsuite.arquillian.jmx.JmxConnectorRegistry;
+import org.keycloak.testsuite.arquillian.undertow.KeycloakOnUndertow;
+import java.io.NotSerializableException;
+import java.lang.management.ManagementFactory;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.jboss.arquillian.core.spi.Validate;
+import org.jboss.logging.Logger;
+
+/**
+ *
+ * @author hmlnarik
+ */
+public class CacheStatisticsControllerEnricher implements TestEnricher {
+
+    private static final Logger LOG = Logger.getLogger(CacheStatisticsControllerEnricher.class);
+
+    @Inject
+    private Instance<ContainerRegistry> registry;
+
+    @Inject
+    private Instance<JmxConnectorRegistry> jmxConnectorRegistry;
+
+    @Inject
+    private Instance<SuiteContext> suiteContext;
+
+    @Override
+    public void enrich(Object testCase) {
+        Validate.notNull(registry.get(), "registry should not be null");
+        Validate.notNull(jmxConnectorRegistry.get(), "jmxConnectorRegistry should not be null");
+        Validate.notNull(suiteContext.get(), "suiteContext should not be null");
+
+        for (Field field : FieldUtils.getAllFields(testCase.getClass())) {
+            JmxInfinispanCacheStatistics annotation = field.getAnnotation(JmxInfinispanCacheStatistics.class);
+
+            if (annotation == null) {
+                continue;
+            }
+
+            try {
+                FieldUtils.writeField(field, testCase, getInfinispanCacheStatistics(annotation), true);
+            } catch (IOException | IllegalAccessException | MalformedObjectNameException e) {
+                throw new RuntimeException("Could not set value on field " + field);
+            }
+        }
+    }
+
+    private InfinispanStatistics getInfinispanCacheStatistics(JmxInfinispanCacheStatistics annotation) throws MalformedObjectNameException, IOException, MalformedURLException {
+        MBeanServerConnection mbsc = getJmxServerConnection(annotation);
+
+        ObjectName mbeanName = new ObjectName(String.format(
+          "%s:type=%s,name=\"%s(%s)\",manager=\"%s\",component=%s",
+          annotation.domain().isEmpty() ? getDefaultDomain(annotation.dcIndex(), annotation.dcNodeIndex()) : InfinispanConnectionProvider.JMX_DOMAIN,
+          annotation.type(),
+          annotation.cacheName(),
+          annotation.cacheMode(),
+          annotation.cacheManagerName(),
+          annotation.component()
+        ));
+
+        InfinispanStatistics value = new InfinispanCacheStatisticsImpl(mbsc, mbeanName);
+
+        if (annotation.domain().isEmpty()) {
+            try {
+                Retry.execute(() -> value.reset(), 2, 150);
+            } catch (RuntimeException ex) {
+                if (annotation.dcIndex() != -1 && annotation.dcNodeIndex() != -1
+                   && suiteContext.get().getAuthServerBackendsInfo(annotation.dcIndex()).get(annotation.dcNodeIndex()).isStarted()) {
+                    LOG.warn("Could not reset statistics for " + mbeanName);
+                }
+            }
+        }
+
+        return value;
+    }
+
+    private InfinispanStatistics getJGroupsChannelStatistics(JmxInfinispanChannelStatistics annotation) throws MalformedObjectNameException, IOException, MalformedURLException {
+        MBeanServerConnection mbsc = getJmxServerConnection(annotation);
+
+        ObjectName mbeanName = new ObjectName(String.format(
+          "%s:type=%s,cluster=\"%s\"",
+          annotation.domain().isEmpty() ? getDefaultDomain(annotation.dcIndex(), annotation.dcNodeIndex()) : InfinispanConnectionProvider.JMX_DOMAIN,
+          annotation.type(),
+          annotation.cluster()
+        ));
+
+        InfinispanStatistics value = new InfinispanChannelStatisticsImpl(mbsc, mbeanName);
+
+        if (annotation.domain().isEmpty()) {
+            try {
+                Retry.execute(() -> value.reset(), 2, 150);
+            } catch (RuntimeException ex) {
+                if (annotation.dcIndex() != -1 && annotation.dcNodeIndex() != -1
+                   && suiteContext.get().getAuthServerBackendsInfo(annotation.dcIndex()).get(annotation.dcNodeIndex()).isStarted()) {
+                    LOG.warn("Could not reset statistics for " + mbeanName);
+                }
+            }
+        }
+
+        return value;
+    }
+
+    @Override
+    public Object[] resolve(Method method) {
+        Object[] values = new Object[method.getParameterCount()];
+
+        for (int i = 0; i < method.getParameterCount(); i ++) {
+            Parameter param = method.getParameters()[i];
+
+            JmxInfinispanCacheStatistics annotation = param.getAnnotation(JmxInfinispanCacheStatistics.class);
+            if (annotation != null) try {
+                values[i] = getInfinispanCacheStatistics(annotation);
+            } catch (IOException | MalformedObjectNameException e) {
+                throw new RuntimeException("Could not set value on field " + param);
+            }
+
+            JmxInfinispanChannelStatistics channelAnnotation = param.getAnnotation(JmxInfinispanChannelStatistics.class);
+            if (channelAnnotation != null) try {
+                values[i] = getJGroupsChannelStatistics(channelAnnotation);
+            } catch (IOException | MalformedObjectNameException e) {
+                throw new RuntimeException("Could not set value on field " + param);
+            }
+        }
+
+        return values;
+    }
+
+    private String getDefaultDomain(int dcIndex, int dcNodeIndex) {
+        if (dcIndex != -1 && dcNodeIndex != -1) {
+            return InfinispanConnectionProvider.JMX_DOMAIN + "-" + suiteContext.get().getAuthServerBackendsInfo(dcIndex).get(dcNodeIndex).getQualifier();
+        }
+        return InfinispanConnectionProvider.JMX_DOMAIN;
+    }
+
+    private MBeanServerConnection getJmxServerConnection(JmxInfinispanCacheStatistics annotation) throws MalformedURLException, IOException {
+        final String host;
+        final int port;
+
+        if (annotation.dcIndex() != -1 && annotation.dcNodeIndex() != -1) {
+            ContainerInfo node = suiteContext.get().getAuthServerBackendsInfo(annotation.dcIndex()).get(annotation.dcNodeIndex());
+            Container container = node.getArquillianContainer();
+            if (container.getDeployableContainer() instanceof KeycloakOnUndertow) {
+                return ManagementFactory.getPlatformMBeanServer();
+            }
+            host = "localhost";
+            port = container.getContainerConfiguration().getContainerProperties().containsKey("managementPort")
+              ? Integer.valueOf(container.getContainerConfiguration().getContainerProperties().get("managementPort"))
+              : 9990;
+        } else {
+            host = annotation.host().isEmpty()
+              ? System.getProperty((annotation.hostProperty().isEmpty()
+                ? "keycloak.connectionsInfinispan.remoteStoreServer"
+                : annotation.hostProperty()))
+              : annotation.host();
+
+            port = annotation.managementPort() == -1
+              ? Integer.valueOf(System.getProperty((annotation.managementPortProperty().isEmpty()
+                ? "cache.server.management.port"
+                : annotation.managementPortProperty())))
+              : annotation.managementPort();
+        }
+
+        JMXServiceURL url = new JMXServiceURL("service:jmx:remote+http://" + host + ":" + port);
+        JMXConnector jmxc = jmxConnectorRegistry.get().getConnection(url);
+
+        return jmxc.getMBeanServerConnection();
+    }
+
+    private MBeanServerConnection getJmxServerConnection(JmxInfinispanChannelStatistics annotation) throws MalformedURLException, IOException {
+        final String host;
+        final int port;
+
+        if (annotation.dcIndex() != -1 && annotation.dcNodeIndex() != -1) {
+            ContainerInfo node = suiteContext.get().getAuthServerBackendsInfo(annotation.dcIndex()).get(annotation.dcNodeIndex());
+            Container container = node.getArquillianContainer();
+            if (container.getDeployableContainer() instanceof KeycloakOnUndertow) {
+                return ManagementFactory.getPlatformMBeanServer();
+            }
+            host = "localhost";
+            port = container.getContainerConfiguration().getContainerProperties().containsKey("managementPort")
+              ? Integer.valueOf(container.getContainerConfiguration().getContainerProperties().get("managementPort"))
+              : 9990;
+        } else {
+            host = annotation.host().isEmpty()
+              ? System.getProperty((annotation.hostProperty().isEmpty()
+                ? "keycloak.connectionsInfinispan.remoteStoreServer"
+                : annotation.hostProperty()))
+              : annotation.host();
+
+            port = annotation.managementPort() == -1
+              ? Integer.valueOf(System.getProperty((annotation.managementPortProperty().isEmpty()
+                ? "cache.server.management.port"
+                : annotation.managementPortProperty())))
+              : annotation.managementPort();
+        }
+
+        JMXServiceURL url = new JMXServiceURL("service:jmx:remote+http://" + host + ":" + port);
+        JMXConnector jmxc = jmxConnectorRegistry.get().getConnection(url);
+
+        return jmxc.getMBeanServerConnection();
+    }
+
+    private static abstract class CacheStatisticsImpl implements InfinispanStatistics {
+
+        protected final MBeanServerConnection mbsc;
+        private final ObjectName mbeanNameTemplate;
+        private ObjectName mbeanName;
+
+        public CacheStatisticsImpl(MBeanServerConnection mbsc, ObjectName mbeanNameTemplate) {
+            this.mbsc = mbsc;
+            this.mbeanNameTemplate = mbeanNameTemplate;
+        }
+
+        @Override
+        public boolean exists() {
+            try {
+                getMbeanName();
+                return true;
+            } catch (Exception ex) {
+                return false;
+            }
+        }
+
+        @Override
+        public Map<String, Object> getStatistics() {
+            try {
+                MBeanInfo mBeanInfo = mbsc.getMBeanInfo(getMbeanName());
+                String[] statAttrs = Arrays.asList(mBeanInfo.getAttributes()).stream()
+                  .filter(MBeanAttributeInfo::isReadable)
+                  .map(MBeanAttributeInfo::getName)
+                  .collect(Collectors.toList())
+                  .toArray(new String[] {});
+                return mbsc.getAttributes(getMbeanName(), statAttrs)
+                  .asList()
+                  .stream()
+                  .collect(Collectors.toMap(Attribute::getName, Attribute::getValue));
+            } catch (IOException | InstanceNotFoundException | ReflectionException | IntrospectionException ex) {
+                throw new RuntimeException(ex);
+            }
+        }
+
+        protected ObjectName getMbeanName() throws IOException, RuntimeException {
+            if (this.mbeanName == null) {
+                Set<ObjectName> queryNames = mbsc.queryNames(mbeanNameTemplate, null);
+                if (queryNames.isEmpty()) {
+                    throw new RuntimeException("No MBean of template " + mbeanNameTemplate + " found at JMX server");
+                }
+                this.mbeanName = queryNames.iterator().next();
+            }
+
+            return this.mbeanName;
+        }
+
+        @Override
+        public Comparable getSingleStatistics(String statisticsName) {
+            try {
+                return (Comparable) mbsc.getAttribute(getMbeanName(), statisticsName);
+            } catch (IOException | InstanceNotFoundException | MBeanException | ReflectionException | AttributeNotFoundException ex) {
+                throw new RuntimeException(ex);
+            }
+        }
+
+        @Override
+        public void waitToBecomeAvailable(int time, TimeUnit unit) {
+            long timeInMillis = TimeUnit.MILLISECONDS.convert(time, unit);
+            Retry.execute(() -> {
+                try {
+                    getMbeanName();
+                    if (! isAvailable()) throw new RuntimeException("Not available");
+                } catch (Exception ex) {
+                    throw new RuntimeException("Timed out while waiting for " + mbeanNameTemplate + " to become available", ex);
+                }
+            }, 1 + (int) timeInMillis / 100, 100);
+        }
+
+        protected abstract boolean isAvailable();
+    }
+
+    private static class InfinispanCacheStatisticsImpl extends CacheStatisticsImpl {
+
+        public InfinispanCacheStatisticsImpl(MBeanServerConnection mbsc, ObjectName mbeanName) {
+            super(mbsc, mbeanName);
+        }
+
+        @Override
+        public void reset() {
+            try {
+                mbsc.invoke(getMbeanName(), "resetStatistics", new Object[] {}, new String[] {});
+            } catch (IOException | InstanceNotFoundException | MBeanException | ReflectionException ex) {
+                throw new RuntimeException(ex);
+            }
+        }
+
+        @Override
+        protected boolean isAvailable() {
+            return getSingleStatistics(Constants.STAT_CACHE_ELAPSED_TIME) != null;
+        }
+    }
+
+    private static class InfinispanChannelStatisticsImpl extends CacheStatisticsImpl {
+
+        public InfinispanChannelStatisticsImpl(MBeanServerConnection mbsc, ObjectName mbeanName) {
+            super(mbsc, mbeanName);
+        }
+
+        @Override
+        public void reset() {
+            try {
+                mbsc.invoke(getMbeanName(), "resetStats", new Object[] {}, new String[] {});
+            } catch (NotSerializableException ex) {
+                // Ignore return value not serializable, the invocation has already done its job
+            } catch (IOException | InstanceNotFoundException | MBeanException | ReflectionException ex) {
+                throw new RuntimeException(ex);
+            }
+        }
+
+        @Override
+        protected boolean isAvailable() {
+            return Objects.equals(getSingleStatistics(Constants.STAT_CHANNEL_CONNECTED), Boolean.TRUE);
+       }
+    }
+}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/InfinispanStatistics.java

@@ -0,0 +1,88 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.arquillian;
+
+import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ *
+ * @author hmlnarik
+ */
+public interface InfinispanStatistics {
+
+    public static class Constants {
+        public static final String DOMAIN_INFINISPAN_DATAGRID = InfinispanConnectionProvider.JMX_DOMAIN;
+
+        public static final String TYPE_CHANNEL = "channel";
+        public static final String TYPE_CACHE = "Cache";
+        public static final String TYPE_CACHE_MANAGER = "CacheManager";
+
+        public static final String COMPONENT_STATISTICS = "Statistics";
+
+        /** Cache statistics */
+        public static final String STAT_CACHE_AVERAGE_READ_TIME = "averageReadTime";
+        public static final String STAT_CACHE_AVERAGE_WRITE_TIME = "averageWriteTime";
+        public static final String STAT_CACHE_ELAPSED_TIME = "elapsedTime";
+        public static final String STAT_CACHE_EVICTIONS = "evictions";
+        public static final String STAT_CACHE_HITS = "hits";
+        public static final String STAT_CACHE_HIT_RATIO = "hitRatio";
+        public static final String STAT_CACHE_MISSES = "misses";
+        public static final String STAT_CACHE_NUMBER_OF_ENTRIES = "numberOfEntries";
+        public static final String STAT_CACHE_NUMBER_OF_ENTRIES_IN_MEMORY = "numberOfEntriesInMemory";
+        public static final String STAT_CACHE_READ_WRITE_RATIO = "readWriteRatio";
+        public static final String STAT_CACHE_REMOVE_HITS = "removeHits";
+        public static final String STAT_CACHE_REMOVE_MISSES = "removeMisses";
+        public static final String STAT_CACHE_STORES = "stores";
+        public static final String STAT_CACHE_TIME_SINCE_RESET = "timeSinceReset";
+
+        /** JGroups channel statistics */
+        public static final String STAT_CHANNEL_ADDRESS = "address";
+        public static final String STAT_CHANNEL_ADDRESS_UUID = "address_uuid";
+        public static final String STAT_CHANNEL_CLOSED = "closed";
+        public static final String STAT_CHANNEL_CLUSTER_NAME = "cluster_name";
+        public static final String STAT_CHANNEL_CONNECTED = "connected";
+        public static final String STAT_CHANNEL_CONNECTING = "connecting";
+        public static final String STAT_CHANNEL_DISCARD_OWN_MESSAGES = "discard_own_messages";
+        public static final String STAT_CHANNEL_OPEN = "open";
+        public static final String STAT_CHANNEL_RECEIVED_BYTES = "received_bytes";
+        public static final String STAT_CHANNEL_RECEIVED_MESSAGES = "received_messages";
+        public static final String STAT_CHANNEL_SENT_BYTES = "sent_bytes";
+        public static final String STAT_CHANNEL_SENT_MESSAGES = "sent_messages";
+        public static final String STAT_CHANNEL_STATE = "state";
+        public static final String STAT_CHANNEL_STATS = "stats";
+        public static final String STAT_CHANNEL_VIEW = "view";
+
+    }
+
+    Map<String, Object> getStatistics();
+
+    Comparable getSingleStatistics(String statisticsName);
+
+    void waitToBecomeAvailable(int time, TimeUnit unit);
+
+    /**
+     * Resets the statistics counters.
+     */
+    void reset();
+
+    /**
+     * Returns {@code true} iff the statistics represented by this object can be retrieved from the server.
+     */
+    boolean exists();
+}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/KeycloakArquillianExtension.java

@@ -32,6 +32,7 @@ import org.jboss.arquillian.graphene.location.CustomizableURLResourceProvider;
 import org.jboss.arquillian.test.spi.enricher.resource.ResourceProvider;
 import org.jboss.arquillian.test.spi.execution.TestExecutionDecider;
 import org.keycloak.testsuite.arquillian.h2.H2TestEnricher;
+import org.keycloak.testsuite.arquillian.jmx.JmxConnectorRegistryCreator;
 import org.keycloak.testsuite.arquillian.karaf.CustomKarafContainer;
 import org.keycloak.testsuite.arquillian.migration.MigrationTestExecutionDecider;
 import org.keycloak.testsuite.arquillian.provider.AdminClientProvider;
@@ -44,6 +45,7 @@ import org.keycloak.testsuite.drone.HtmlUnitScreenshots;
 import org.keycloak.testsuite.drone.KeycloakDronePostSetup;
 import org.keycloak.testsuite.drone.KeycloakHtmlUnitInstantiator;
 import org.keycloak.testsuite.drone.KeycloakWebDriverConfigurator;
+import org.jboss.arquillian.test.spi.TestEnricher;
 
 /**
  *
@@ -65,6 +67,8 @@ public class KeycloakArquillianExtension implements LoadableExtension {
                 .service(DeploymentScenarioGenerator.class, DeploymentTargetModifier.class)
                 .service(ApplicationArchiveProcessor.class, DeploymentArchiveProcessor.class)
                 .service(DeployableContainer.class, CustomKarafContainer.class)
+                .service(TestEnricher.class, CacheStatisticsControllerEnricher.class)
+                .observer(JmxConnectorRegistryCreator.class)
                 .observer(AuthServerTestEnricher.class)
                 .observer(AppServerTestEnricher.class)
                 .observer(H2TestEnricher.class);

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/annotation/JmxInfinispanCacheStatistics.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.arquillian.annotation;
+
+import org.keycloak.testsuite.arquillian.AuthServerTestEnricher;
+import org.keycloak.testsuite.arquillian.InfinispanStatistics;
+import org.keycloak.testsuite.arquillian.InfinispanStatistics.Constants;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation for a field / method parameter annotating {@link InfinispanStatistics} object that would be used
+ * to access statistics via JMX. By default, the access to "work" cache at remote infinispan / JDG server is requested
+ * yet the same annotation is used for other caches as well.
+ *
+ * @author hmlnarik
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.FIELD, ElementType.PARAMETER})
+public @interface JmxInfinispanCacheStatistics {
+
+    /** JMX domain. Should be set to default (@{code ""}) if the node to get the statistics from should be obtained from {@link #dcIndex()} and {@link #dcNodeIndex()}. */
+    String domain() default "";
+
+    // JMX address properties
+    String type() default Constants.TYPE_CACHE;
+    String cacheName() default "work";
+    String cacheMode() default "*";
+    String cacheManagerName() default "*";
+    String component() default Constants.COMPONENT_STATISTICS;
+
+    // Host address - either given by arrangement of DC ...
+
+    /** Index of the data center, starting from 0 */
+    int dcIndex() default -1;
+    /** Index of the node within data center, starting from 0. Nodes are ordered by arquillian qualifier as per {@link AuthServerTestEnricher} */
+    int dcNodeIndex() default -1;
+
+    // ... or by specific host/port
+
+    /** Port for management */
+    int managementPort() default -1;
+    /** Name of system property to obtain management port from */
+    String managementPortProperty() default "";
+    /** Host name to connect to */
+    String host() default "";
+    /** Name of system property to obtain host name from */
+    String hostProperty() default "";
+
+}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/annotation/JmxInfinispanChannelStatistics.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.arquillian.annotation;
+
+import org.keycloak.testsuite.arquillian.InfinispanStatistics.Constants;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ *
+ * @author hmlnarik
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.FIELD, ElementType.PARAMETER})
+public @interface JmxInfinispanChannelStatistics {
+
+    /** JMX domain. Should be set to default (@{code ""}) if the node to get the statistics from should be obtained from {@link #dcIndex()} and {@link #dcNodeIndex()}. */
+    String domain() default "";
+
+    // JMX address properties
+    String type() default Constants.TYPE_CHANNEL;
+    String cluster() default "*";
+
+    // Host address - either given by arrangement of DC ...
+
+    /** Index of the data center, starting from 0 */
+    int dcIndex() default -1;
+    /** Index of the node within data center, starting from 0. Nodes are ordered by arquillian qualifier as per {@link AuthServerTestEnricher} */
+    int dcNodeIndex() default -1;
+
+    /** Port for management */
+    int managementPort() default -1;
+    /** Name of system property to obtain management port from */
+    String managementPortProperty() default "";
+    /** Host name to connect to */
+    String host() default "";
+    /** Name of system property to obtain host name from */
+    String hostProperty() default "";
+}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/containers/RegistryCreator.java

@@ -35,6 +35,7 @@ import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 
+import org.mvel2.MVEL;
 import static org.keycloak.testsuite.arquillian.containers.SecurityActions.isClassPresent;
 import static org.keycloak.testsuite.arquillian.containers.SecurityActions.loadClass;
 
@@ -97,10 +98,14 @@ public class RegistryCreator {
 
     private static final String ENABLED = "enabled";
 
-    private boolean isEnabled(ContainerDef containerDef) {
+    private static boolean isEnabled(ContainerDef containerDef) {
         Map<String, String> props = containerDef.getContainerProperties();
-        return !props.containsKey(ENABLED)
+        try {
-                || (props.containsKey(ENABLED) && props.get(ENABLED).equals("true"));
+            return !props.containsKey(ENABLED)
+                    || (props.containsKey(ENABLED) && ! props.get(ENABLED).isEmpty() && MVEL.evalToBoolean(props.get(ENABLED), (Object) null));
+        } catch (Exception ex) {
+            return false;
+        }
     }
 
     @SuppressWarnings("rawtypes")

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/jmx/JmxConnectorRegistry.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.arquillian.jmx;
+
+import javax.management.remote.JMXConnector;
+import javax.management.remote.JMXServiceURL;
+
+/**
+ *
+ * @author hmlnarik
+ */
+public interface JmxConnectorRegistry {
+    JMXConnector getConnection(JMXServiceURL url);
+
+    void closeAll();
+}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/jmx/JmxConnectorRegistryCreator.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.arquillian.jmx;
+
+import java.io.IOException;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import javax.management.remote.JMXConnector;
+import javax.management.remote.JMXConnectorFactory;
+import javax.management.remote.JMXServiceURL;
+import org.jboss.arquillian.core.api.InstanceProducer;
+import org.jboss.arquillian.core.api.annotation.ApplicationScoped;
+import org.jboss.arquillian.core.api.annotation.Inject;
+import org.jboss.arquillian.core.api.annotation.Observes;
+import org.jboss.arquillian.test.spi.event.suite.BeforeSuite;
+
+/**
+ *
+ * @author hmlnarik
+ */
+public class JmxConnectorRegistryCreator {
+
+    @Inject
+    @ApplicationScoped
+    private InstanceProducer<JmxConnectorRegistry> connectorRegistry;
+
+    public void configureJmxConnectorRegistry(@Observes BeforeSuite event) {
+        if (connectorRegistry.get() == null) {
+            connectorRegistry.set(new JmxConnectorRegistry() {
+
+                private volatile ConcurrentMap<JMXServiceURL, JMXConnector> connectors = new ConcurrentHashMap<>();
+
+                @Override
+                public JMXConnector getConnection(JMXServiceURL url) {
+                    JMXConnector res = connectors.get(url);
+                    if (res == null) {
+                        try {
+                            final JMXConnector conn = JMXConnectorFactory.newJMXConnector(url, null);
+                            res = connectors.putIfAbsent(url, conn);
+                            if (res == null) {
+                                res = conn;
+                            }
+                            res.connect();
+                        } catch (IOException ex) {
+                            throw new RuntimeException("Could not instantiate JMX connector for " + url, ex);
+                        }
+                    }
+                    return res;
+                }
+
+                @Override
+                public void closeAll() {
+                    connectors.values().forEach(c -> { try { c.close(); } catch (IOException e) {} });
+                    connectors.clear();
+                }
+            });
+        }
+    }
+}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/provider/LoadBalancerControllerProvider.java

@@ -2,14 +2,8 @@ package org.keycloak.testsuite.arquillian.provider;
 
 import org.keycloak.testsuite.arquillian.annotation.LoadBalancer;
 import java.lang.annotation.Annotation;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import org.jboss.arquillian.container.spi.event.KillContainer;
-import org.jboss.arquillian.container.spi.event.StartContainer;
-import org.jboss.arquillian.container.spi.event.StopContainer;
 import org.jboss.arquillian.core.api.Instance;
 import org.jboss.arquillian.core.api.annotation.Inject;
-import org.jboss.arquillian.core.api.annotation.Observes;
 import org.jboss.arquillian.test.api.ArquillianResource;
 import org.jboss.arquillian.test.spi.enricher.resource.ResourceProvider;
 import org.keycloak.testsuite.arquillian.LoadBalancerController;

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractAdminCrossDCTest.java

@@ -19,13 +19,20 @@ package org.keycloak.testsuite.crossdc;
 import org.keycloak.admin.client.resource.RealmResource;
 import org.keycloak.events.log.JBossLoggingEventListenerProviderFactory;
 import org.keycloak.representations.idm.RealmRepresentation;
+import org.keycloak.testsuite.Retry;
+import org.keycloak.testsuite.arquillian.InfinispanStatistics;
 import org.keycloak.testsuite.events.EventsListenerProviderFactory;
 import org.keycloak.testsuite.util.TestCleanup;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import org.hamcrest.Matcher;
 import org.junit.Before;
+import static org.junit.Assert.assertThat;
 
 /**
  *
@@ -78,4 +85,31 @@ public abstract class AbstractAdminCrossDCTest extends AbstractCrossDCTest {
     protected TestCleanup getCleanup() {
         return getCleanup(REALM_NAME);
     }
+
+    protected <T extends Comparable> void assertSingleStatistics(InfinispanStatistics stats, String key, Runnable testedCode, Function<T, Matcher<? super T>> matcherOnOldStat) {
+        stats.reset();
+
+        T oldStat = (T) stats.getSingleStatistics(key);
+        testedCode.run();
+
+        Retry.execute(() -> {
+            T newStat = (T) stats.getSingleStatistics(key);
+
+            Matcher<? super T> matcherInstance = matcherOnOldStat.apply(oldStat);
+            assertThat(newStat, matcherInstance);
+        }, 5, 200);
+    }
+
+    protected void assertStatistics(InfinispanStatistics stats, Runnable testedCode, BiConsumer<Map<String, Object>, Map<String, Object>> assertionOnStats) {
+        stats.reset();
+
+        Map<String, Object> oldStat = stats.getStatistics();
+        testedCode.run();
+
+        Retry.execute(() -> {
+            Map<String, Object> newStat = stats.getStatistics();
+            assertionOnStats.accept(oldStat, newStat);
+        }, 5, 200);
+    }
+
 }

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractCrossDCTest.java

@@ -32,14 +32,21 @@ import org.jboss.arquillian.test.api.ArquillianResource;
 import org.junit.After;
 import org.junit.Before;
 
+import static org.hamcrest.Matchers.lessThan;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
 /**
  *
  * @author hmlnarik
  */
 public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest {
 
+    // Keep the following constants in sync with arquillian
+    public static final String QUALIFIER_NODE_BALANCER = "auth-server-balancer-cross-dc";
+
     @ArquillianResource
-    @LoadBalancer(value = "auth-server-balancer-cross-dc")
+    @LoadBalancer(value = QUALIFIER_NODE_BALANCER)
     protected LoadBalancerController loadBalancerCtrl;
 
     @ArquillianResource
@@ -103,6 +110,11 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
         return Keycloak.getInstance(node.getContextRoot() + "/auth", AuthRealm.MASTER, AuthRealm.ADMIN, AuthRealm.ADMIN, Constants.ADMIN_CLI_CLIENT_ID);
     }
 
+    /**
+     * Creates admin client directed to the given node.
+     * @param node
+     * @return
+     */
     protected Keycloak getAdminClientFor(ContainerInfo node) {
         Keycloak adminClient = backendAdminClients.get(node);
         if (adminClient == null && node.equals(suiteContext.getAuthServerInfo())) {
@@ -111,13 +123,17 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
         return adminClient;
     }
 
+    /**
+     * Disables routing requests to the given data center in the load balancer.
+     * @param dcIndex
+     */
     public void disableDcOnLoadBalancer(int dcIndex) {
         log.infof("Disabling load balancer for dc=%d", dcIndex);
         this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex).forEach(c -> loadBalancerCtrl.disableBackendNodeByName(c.getQualifier()));
     }
 
     /**
-     * Enables all started nodes in the given data center
+     * Enables routing requests to all started nodes to the given data center in the load balancer.
      * @param dcIndex
      */
     public void enableDcOnLoadBalancer(int dcIndex) {
@@ -132,11 +148,21 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
         }
     }
 
+    /**
+     * Disables routing requests to the given node within the given data center in the load balancer.
+     * @param dcIndex
+     * @param nodeIndex
+     */
     public void disableLoadBalancerNode(int dcIndex, int nodeIndex) {
         log.infof("Disabling load balancer for dc=%d, node=%d", dcIndex, nodeIndex);
         loadBalancerCtrl.disableBackendNodeByName(this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex).get(nodeIndex).getQualifier());
     }
 
+    /**
+     * Enables routing requests to the given node within the given data center in the load balancer.
+     * @param dcIndex
+     * @param nodeIndex
+     */
     public void enableLoadBalancerNode(int dcIndex, int nodeIndex) {
         log.infof("Enabling load balancer for dc=%d, node=%d", dcIndex, nodeIndex);
         final ContainerInfo backendNode = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex).get(nodeIndex);
@@ -149,11 +175,53 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
         loadBalancerCtrl.enableBackendNodeByName(backendNode.getQualifier());
     }
 
+    /**
+     * Starts a manually-controlled backend auth-server node in cross-DC scenario.
+     * @param dcIndex
+     * @param nodeIndex
+     * @return Started instance descriptor.
+     */
+    public ContainerInfo startBackendNode(int dcIndex, int nodeIndex) {
+        assertThat((Integer) dcIndex, lessThan(this.suiteContext.getDcAuthServerBackendsInfo().size()));
+        final List<ContainerInfo> dcNodes = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex);
+        assertThat((Integer) nodeIndex, lessThan(dcNodes.size()));
+        ContainerInfo dcNode = dcNodes.get(nodeIndex);
+        assertTrue("Node " + dcNode.getQualifier() + " has to be controlled manually", dcNode.isManual());
+        containerController.start(dcNode.getQualifier());
+        return dcNode;
+    }
+
+    /**
+     * Stops a manually-controlled backend auth-server node in cross-DC scenario.
+     * @param dcIndex
+     * @param nodeIndex
+     * @return Stopped instance descriptor.
+     */
+    public ContainerInfo stopBackendNode(int dcIndex, int nodeIndex) {
+        assertThat((Integer) dcIndex, lessThan(this.suiteContext.getDcAuthServerBackendsInfo().size()));
+        final List<ContainerInfo> dcNodes = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex);
+        assertThat((Integer) nodeIndex, lessThan(dcNodes.size()));
+        ContainerInfo dcNode = dcNodes.get(nodeIndex);
+        assertTrue("Node " + dcNode.getQualifier() + " has to be controlled manually", dcNode.isManual());
+        containerController.stop(dcNode.getQualifier());
+        return dcNode;
+    }
+
+    /**
+     * Returns stream of all nodes in the given dc that are started manually.
+     * @param dcIndex
+     * @return
+     */
     public Stream<ContainerInfo> getManuallyStartedBackendNodes(int dcIndex) {
         final List<ContainerInfo> dcNodes = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex);
         return dcNodes.stream().filter(ContainerInfo::isManual);
     }
 
+    /**
+     * Returns stream of all nodes in the given dc that are started automatically.
+     * @param dcIndex
+     * @return
+     */
     public Stream<ContainerInfo> getAutomaticallyStartedBackendNodes(int dcIndex) {
         final List<ContainerInfo> dcNodes = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex);
         return dcNodes.stream().filter(c -> ! c.isManual());

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/ActionTokenCrossDCTest.java

@@ -17,16 +17,13 @@
 package org.keycloak.testsuite.crossdc;
 
 import org.keycloak.admin.client.resource.UserResource;
-import org.keycloak.events.admin.OperationType;
+import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
-import org.keycloak.events.admin.ResourceType;
 import org.keycloak.models.UserModel;
 import org.keycloak.representations.idm.UserRepresentation;
 import org.keycloak.testsuite.Retry;
 import org.keycloak.testsuite.admin.ApiUtil;
-import org.keycloak.testsuite.arquillian.ContainerInfo;
 import org.keycloak.testsuite.page.LoginPasswordUpdatePage;
 import org.keycloak.testsuite.pages.ErrorPage;
-import org.keycloak.testsuite.util.AdminEventPaths;
 import org.keycloak.testsuite.util.GreenMailRule;
 import org.keycloak.testsuite.util.MailUtils;
 import java.io.IOException;
@@ -36,12 +33,20 @@ import javax.mail.MessagingException;
 import javax.mail.internet.MimeMessage;
 import javax.ws.rs.core.Response;
 import org.jboss.arquillian.graphene.page.Page;
-import org.jboss.arquillian.test.api.ArquillianResource;
 import org.junit.Assert;
 import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import static org.junit.Assert.assertEquals;
+import org.keycloak.testsuite.arquillian.annotation.JmxInfinispanCacheStatistics;
+import org.keycloak.testsuite.arquillian.annotation.JmxInfinispanChannelStatistics;
+import org.keycloak.testsuite.arquillian.InfinispanStatistics;
+import org.keycloak.testsuite.arquillian.InfinispanStatistics.Constants;
+import java.util.concurrent.TimeUnit;
+import org.hamcrest.Matchers;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
 
 /**
  *
@@ -69,7 +74,16 @@ public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest {
     }
 
     @Test
-    public void sendResetPasswordEmailSuccessWorksInCrossDc() throws IOException, MessagingException {
+    public void sendResetPasswordEmailSuccessWorksInCrossDc(
+      @JmxInfinispanCacheStatistics(dcIndex=0, dcNodeIndex=0, cacheName=InfinispanConnectionProvider.ACTION_TOKEN_CACHE) InfinispanStatistics cacheDc0Node0Statistics,
+      @JmxInfinispanCacheStatistics(dcIndex=0, dcNodeIndex=1, cacheName=InfinispanConnectionProvider.ACTION_TOKEN_CACHE) InfinispanStatistics cacheDc0Node1Statistics,
+      @JmxInfinispanCacheStatistics(dcIndex=1, dcNodeIndex=0, cacheName=InfinispanConnectionProvider.ACTION_TOKEN_CACHE) InfinispanStatistics cacheDc1Node0Statistics,
+      @JmxInfinispanChannelStatistics() InfinispanStatistics channelStatisticsCrossDc) throws Exception {
+        startBackendNode(0, 1);
+        cacheDc0Node1Statistics.waitToBecomeAvailable(10, TimeUnit.SECONDS);
+
+        Comparable originalNumberOfEntries = cacheDc0Node0Statistics.getSingleStatistics(Constants.STAT_CACHE_NUMBER_OF_ENTRIES);
+
         UserRepresentation userRep = new UserRepresentation();
         userRep.setEnabled(true);
         userRep.setUsername("user1");
@@ -88,21 +102,33 @@ public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest {
 
         String link = MailUtils.getPasswordResetEmailLink(message);
 
-        driver.navigate().to(link);
+        Retry.execute(() -> channelStatisticsCrossDc.reset(), 3, 100);
+
+        assertSingleStatistics(cacheDc0Node0Statistics, Constants.STAT_CACHE_NUMBER_OF_ENTRIES,
+          () -> driver.navigate().to(link),
+          Matchers::is
+        );
 
         passwordUpdatePage.assertCurrent();
 
-        passwordUpdatePage.changePassword("new-pass", "new-pass");
+        // Verify that there was at least one message sent via the channel
+        assertSingleStatistics(channelStatisticsCrossDc, Constants.STAT_CHANNEL_SENT_MESSAGES,
+          () -> passwordUpdatePage.changePassword("new-pass", "new-pass"),
+          old -> greaterThan((Comparable) 0l)
+        );
+
+        // Verify that the caches are synchronized
+        assertThat(cacheDc0Node0Statistics.getSingleStatistics(Constants.STAT_CACHE_NUMBER_OF_ENTRIES), greaterThan(originalNumberOfEntries));
+        assertThat(cacheDc0Node0Statistics.getSingleStatistics(Constants.STAT_CACHE_NUMBER_OF_ENTRIES),
+                is(cacheDc1Node0Statistics.getSingleStatistics(Constants.STAT_CACHE_NUMBER_OF_ENTRIES)));
 
         assertEquals("Your account has been updated.", driver.getTitle());
 
         disableDcOnLoadBalancer(0);
         enableDcOnLoadBalancer(1);
 
-        Retry.execute(() -> {
+        driver.navigate().to(link);
-            driver.navigate().to(link);
+        errorPage.assertCurrent();
-            errorPage.assertCurrent();
-        }, 3, 400);
     }
 
     @Ignore("KEYCLOAK-5030")
@@ -144,9 +170,10 @@ public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest {
               loadBalancerCtrl.enableBackendNodeByName(c.getQualifier());
           });
 
-        driver.navigate().to(link);
+        Retry.execute(() -> {
-
+            driver.navigate().to(link);
-        errorPage.assertCurrent();
+            errorPage.assertCurrent();
+        }, 3, 400);
     }
 
 }

testsuite/integration-arquillian/tests/base/src/test/resources/arquillian.xml

@@ -49,7 +49,7 @@
     
     <container qualifier="auth-server-undertow" mode="suite" >
         <configuration>
-            <property name="enabled">${auth.server.undertow}</property>
+            <property name="enabled">${auth.server.undertow} &amp;&amp; ! ${auth.server.undertow.crossdc}</property>
             <property name="bindAddress">localhost</property>
             <property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.KeycloakOnUndertow</property>
             <property name="bindHttpPort">${auth.server.http.port}</property>
@@ -169,12 +169,12 @@
 
     <!-- Cross DC with embedded undertow. Node numbering is [centre #].[node #] -->
     <group qualifier="auth-server-undertow-cross-dc">
-        <container qualifier="cache-server-cross-dc" mode="suite" >
+        <container qualifier="cache-server-cross-dc-1" mode="suite" >
             <configuration>
                 <property name="enabled">${auth.server.undertow.crossdc}</property>
                 <property name="adapterImplClass">org.jboss.as.arquillian.container.managed.ManagedDeployableContainer</property>
                 <property name="jbossHome">${cache.server.home}</property>
-                <property name="serverConfig">standalone.xml</property>
+                <property name="serverConfig">clustered.xml</property>
                 <property name="jbossArguments">
                     -Djboss.socket.binding.port-offset=${cache.server.port.offset}
                     -Djboss.default.multicast.address=234.56.78.99
@@ -192,30 +192,54 @@
             </configuration>
         </container>
 
+        <container qualifier="cache-server-cross-dc-2" mode="suite" >
+            <configuration>
+                <property name="enabled">${auth.server.undertow.crossdc}</property>
+                <property name="adapterImplClass">org.jboss.as.arquillian.container.managed.ManagedDeployableContainer</property>
+                <property name="jbossHome">${cache.server.home}</property>
+                <property name="setupCleanServerBaseDir">true</property>
+                <property name="cleanServerBaseDir">${cache.server.home}/standalone-dc-2</property>
+                <property name="serverConfig">clustered.xml</property>
+                <property name="jbossArguments">
+                    -Djboss.socket.binding.port-offset=${cache.server.2.port.offset}
+                    -Djboss.default.multicast.address=234.56.78.99
+                    -Djboss.node.name=cache-server-dc-2
+                    ${adapter.test.props}
+                    ${auth.server.profile}
+                </property>
+                <property name="javaVmArguments">
+                    ${auth.server.memory.settings}
+                    -Djava.net.preferIPv4Stack=true
+                </property>
+                <property name="outputToConsole">${cache.server.console.output}</property>
+                <property name="managementPort">${cache.server.2.management.port}</property>
+                <property name="startupTimeoutInSeconds">${auth.server.jboss.startup.timeout}</property>
+            </configuration>
+        </container>
+
         <container qualifier="auth-server-balancer-cross-dc" mode="suite" >
             <configuration>
                 <property name="enabled">${auth.server.undertow.crossdc}</property>
                 <property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.lb.SimpleUndertowLoadBalancerContainer</property>
                 <property name="bindAddress">localhost</property>
                 <property name="bindHttpPort">${auth.server.http.port}</property>
-                <property name="bindHttpPortOffset">5</property>
                 <property name="nodes">auth-server-undertow-cross-dc-0.1=http://localhost:8101,auth-server-undertow-cross-dc-0.2-manual=http://localhost:8102,auth-server-undertow-cross-dc-1.1=http://localhost:8111,auth-server-undertow-cross-dc-1.2-manual=http://localhost:8112</property>
             </configuration>
         </container>
 
-        <container qualifier="auth-server-undertow-cross-dc-0.1" mode="suite" >
+        <container qualifier="auth-server-undertow-cross-dc-0_1" mode="suite" >
             <configuration>
                 <property name="enabled">${auth.server.undertow.crossdc}</property>
                 <property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.KeycloakOnUndertow</property>
                 <property name="bindAddress">localhost</property>
                 <property name="bindHttpPort">${auth.server.http.port}</property>
                 <property name="bindHttpPortOffset">-79</property>
-                <property name="route">auth-server-undertow-cross-dc-0.1</property>
+                <property name="route">auth-server-undertow-cross-dc-0_1</property>
                 <property name="remoteMode">${undertow.remote}</property>
                 <property name="dataCenter">0</property>
                 <property name="keycloakConfigPropertyOverrides">{
                     "keycloak.connectionsInfinispan.jgroupsUdpMcastAddr": "234.56.78.1",
-                    "keycloak.connectionsInfinispan.nodeName": "auth-server-undertow-cross-dc-0.1",
+                    "keycloak.connectionsInfinispan.nodeName": "auth-server-undertow-cross-dc-0_1",
                     "keycloak.connectionsInfinispan.clustered": "${keycloak.connectionsInfinispan.clustered:true}",
                     "keycloak.connectionsInfinispan.remoteStoreServer": "${keycloak.connectionsInfinispan.remoteStoreServer:localhost}",
                     "keycloak.connectionsInfinispan.remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort:11222}",
@@ -226,19 +250,19 @@
                 }</property>
             </configuration>
         </container>
-        <container qualifier="auth-server-undertow-cross-dc-0.2-manual" mode="manual" >
+        <container qualifier="auth-server-undertow-cross-dc-0_2-manual" mode="manual" >
             <configuration>
                 <property name="enabled">${auth.server.undertow.crossdc}</property>
                 <property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.KeycloakOnUndertow</property>
                 <property name="bindAddress">localhost</property>
                 <property name="bindHttpPort">${auth.server.http.port}</property>
                 <property name="bindHttpPortOffset">-78</property>
-                <property name="route">auth-server-undertow-cross-dc-0.2</property>
+                <property name="route">auth-server-undertow-cross-dc-0_2-manual</property>
                 <property name="remoteMode">${undertow.remote}</property>
                 <property name="dataCenter">0</property>
                 <property name="keycloakConfigPropertyOverrides">{
                     "keycloak.connectionsInfinispan.jgroupsUdpMcastAddr": "234.56.78.1",
-                    "keycloak.connectionsInfinispan.nodeName": "auth-server-undertow-cross-dc-0.2",
+                    "keycloak.connectionsInfinispan.nodeName": "auth-server-undertow-cross-dc-0_2-manual",
                     "keycloak.connectionsInfinispan.clustered": "${keycloak.connectionsInfinispan.clustered:true}",
                     "keycloak.connectionsInfinispan.remoteStoreServer": "${keycloak.connectionsInfinispan.remoteStoreServer:localhost}",
                     "keycloak.connectionsInfinispan.remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort:11222}",
@@ -250,22 +274,22 @@
             </configuration>
         </container>
 
-        <container qualifier="auth-server-undertow-cross-dc-1.1" mode="suite" >
+        <container qualifier="auth-server-undertow-cross-dc-1_1" mode="suite" >
             <configuration>
                 <property name="enabled">${auth.server.undertow.crossdc}</property>
                 <property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.KeycloakOnUndertow</property>
                 <property name="bindAddress">localhost</property>
                 <property name="bindHttpPort">${auth.server.http.port}</property>
                 <property name="bindHttpPortOffset">-69</property>
-                <property name="route">auth-server-undertow-cross-dc-1.1</property>
+                <property name="route">auth-server-undertow-cross-dc-1_1</property>
                 <property name="remoteMode">${undertow.remote}</property>
                 <property name="dataCenter">1</property>
                 <property name="keycloakConfigPropertyOverrides">{
                     "keycloak.connectionsInfinispan.jgroupsUdpMcastAddr": "234.56.78.2",
-                    "keycloak.connectionsInfinispan.nodeName": "auth-server-undertow-cross-dc-1.1",
+                    "keycloak.connectionsInfinispan.nodeName": "auth-server-undertow-cross-dc-1_1",
                     "keycloak.connectionsInfinispan.clustered": "${keycloak.connectionsInfinispan.clustered:true}",
                     "keycloak.connectionsInfinispan.remoteStoreServer": "${keycloak.connectionsInfinispan.remoteStoreServer:localhost}",
-                    "keycloak.connectionsInfinispan.remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort:11222}",
+                    "keycloak.connectionsInfinispan.remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort.2:11222}",
                     "keycloak.connectionsInfinispan.remoteStoreEnabled": "${keycloak.connectionsInfinispan.remoteStoreEnabled:true}",
                     "keycloak.connectionsJpa.url": "${keycloak.connectionsJpa.url.crossdc:jdbc:h2:mem:test-dc-shared}",
                     "keycloak.connectionsJpa.driver": "${keycloak.connectionsJpa.driver.crossdc:org.h2.Driver}",
@@ -273,22 +297,22 @@
                 }</property>
             </configuration>
         </container>
-        <container qualifier="auth-server-undertow-cross-dc-1.2-manual" mode="manual" >
+        <container qualifier="auth-server-undertow-cross-dc-1_2-manual" mode="manual" >
             <configuration>
                 <property name="enabled">${auth.server.undertow.crossdc}</property>
                 <property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.KeycloakOnUndertow</property>
                 <property name="bindAddress">localhost</property>
                 <property name="bindHttpPort">${auth.server.http.port}</property>
                 <property name="bindHttpPortOffset">-68</property>
-                <property name="route">auth-server-undertow-cross-dc-1.2</property>
+                <property name="route">auth-server-undertow-cross-dc-1_2-manual</property>
                 <property name="remoteMode">${undertow.remote}</property>
                 <property name="dataCenter">1</property>
                 <property name="keycloakConfigPropertyOverrides">{
                     "keycloak.connectionsInfinispan.jgroupsUdpMcastAddr": "234.56.78.2",
-                    "keycloak.connectionsInfinispan.nodeName": "auth-server-undertow-cross-dc-1.2",
+                    "keycloak.connectionsInfinispan.nodeName": "auth-server-undertow-cross-dc-1_2-manual",
                     "keycloak.connectionsInfinispan.clustered": "${keycloak.connectionsInfinispan.clustered:true}",
                     "keycloak.connectionsInfinispan.remoteStoreServer": "${keycloak.connectionsInfinispan.remoteStoreServer:localhost}",
-                    "keycloak.connectionsInfinispan.remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort:11222}",
+                    "keycloak.connectionsInfinispan.remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort.2:11222}",
                     "keycloak.connectionsInfinispan.remoteStoreEnabled": "${keycloak.connectionsInfinispan.remoteStoreEnabled:true}",
                     "keycloak.connectionsJpa.url": "${keycloak.connectionsJpa.url.crossdc:jdbc:h2:mem:test-dc-shared}",
                     "keycloak.connectionsJpa.driver": "${keycloak.connectionsJpa.driver.crossdc:org.h2.Driver}",

testsuite/integration-arquillian/tests/pom.xml

@@ -73,9 +73,12 @@
         <cache.server.home>${containers.home}/${cache.server.container}</cache.server.home>
         <cache.server.port.offset>1010</cache.server.port.offset>
         <cache.server.management.port>11000</cache.server.management.port>
+        <cache.server.2.port.offset>2010</cache.server.2.port.offset>
+        <cache.server.2.management.port>12000</cache.server.2.management.port>
         <cache.server.console.output>true</cache.server.console.output>
         <keycloak.connectionsInfinispan.remoteStoreServer>localhost</keycloak.connectionsInfinispan.remoteStoreServer>
         <keycloak.connectionsInfinispan.remoteStorePort>12232</keycloak.connectionsInfinispan.remoteStorePort>
+        <keycloak.connectionsInfinispan.remoteStorePort.2>13232</keycloak.connectionsInfinispan.remoteStorePort.2>
         <keycloak.connectionsJpa.url.crossdc>jdbc:h2:mem:test-dc-shared</keycloak.connectionsJpa.url.crossdc>
 
         <adapter.test.props/>
@@ -176,6 +179,23 @@
                         </execution>
                     </executions>
                 </plugin>
+                <plugin>
+                    <artifactId>maven-antrun-plugin</artifactId>
+                    <executions>
+                        <execution>
+                            <id>clean-second-cache-server-arquillian-bug-workaround</id>
+                            <phase>process-test-resources</phase>
+                            <goals><goal>run</goal></goals>
+                            <configuration>
+                                <target>
+                                    <echo>${cache.server.home}/standalone-dc-2</echo>
+                                    <delete failonerror="false" dir="${cache.server.home}/standalone-dc-2" />
+                                    <mkdir dir="${cache.server.home}/standalone-dc-2/deployments" />
+                                </target>
+                            </configuration>
+                        </execution>
+                    </executions>
+                </plugin>
                 <plugin>
                     <artifactId>maven-surefire-plugin</artifactId>
                     <configuration>
@@ -252,8 +272,11 @@
                             <cache.server.home>${cache.server.home}</cache.server.home>
                             <cache.server.console.output>${cache.server.console.output}</cache.server.console.output>
                             <cache.server.management.port>${cache.server.management.port}</cache.server.management.port>
+                            <cache.server.2.port.offset>${cache.server.2.port.offset}</cache.server.2.port.offset>
+                            <cache.server.2.management.port>${cache.server.2.management.port}</cache.server.2.management.port>
 
                             <keycloak.connectionsInfinispan.remoteStorePort>${keycloak.connectionsInfinispan.remoteStorePort}</keycloak.connectionsInfinispan.remoteStorePort>
+                            <keycloak.connectionsInfinispan.remoteStorePort.2>${keycloak.connectionsInfinispan.remoteStorePort.2}</keycloak.connectionsInfinispan.remoteStorePort.2>
                             <keycloak.connectionsInfinispan.remoteStoreServer>${keycloak.connectionsInfinispan.remoteStoreServer}</keycloak.connectionsInfinispan.remoteStoreServer>
 
                             <keycloak.connectionsJpa.url.crossdc>${keycloak.connectionsJpa.url.crossdc}</keycloak.connectionsJpa.url.crossdc>