KEYCLOAK-5618 Fix SessionsPreloadCrossDCTest. Update HOW-TO-RUN docs. Ensure it's executed in travis.

misc/CrossDataCenter.md

@@ -98,7 +98,9 @@ Infinispan Server setup
             <transaction mode="NON_DURABLE_XA" locking="PESSIMISTIC"/>
             <locking acquire-timeout="0" />
             <backups>
-                <backup site="site2" failure-policy="FAIL" strategy="SYNC" enabled="true"/>
+                <backup site="site2" failure-policy="FAIL" strategy="SYNC" enabled="true">
+                    <take-offline min-wait="60000" after-failures="3" />
+                </backup>
             </backups>
         </replicated-cache-configuration>
 

misc/Testsuite.md

@@ -117,7 +117,7 @@ But additionally you can enable Kerberos authentication in LDAP provider with th
 * KeyTab: $KEYCLOAK_SOURCES/testsuite/integration-arquillian/tests/base/src/test/resources/kerberos/http.keytab (Replace $KEYCLOAK_SOURCES with correct absolute path of your sources)
 
 Once you do this, you should also ensure that your Kerberos client configuration file is properly configured with KEYCLOAK.ORG domain. 
-See [../testsuite/integration-arquillian/src/test/resources/kerberos/test-krb5.conf](../testsuite/integration-arquillian/src/test/resources/kerberos/test-krb5.conf) for inspiration. The location of Kerberos configuration file 
+See [../testsuite/integration-arquillian/tests/base/src/test/resources/kerberos/test-krb5.conf](../testsuite/integration-arquillian/tests/base/src/test/resources/kerberos/test-krb5.conf) for inspiration. The location of Kerberos configuration file 
 is platform dependent (In linux it's file `/etc/krb5.conf` )
 
 Then you need to configure your browser to allow SPNEGO/Kerberos login from `localhost` .

model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanClusterProvider.java

@@ -22,6 +22,7 @@ import org.keycloak.cluster.ClusterEvent;
 import org.keycloak.cluster.ClusterListener;
 import org.keycloak.cluster.ClusterProvider;
 import org.keycloak.cluster.ExecutionResult;
+import org.keycloak.common.util.Retry;
 import org.keycloak.common.util.Time;
 
 import java.util.concurrent.Callable;
@@ -140,7 +141,7 @@ public class InfinispanClusterProvider implements ClusterProvider {
     private boolean tryLock(String cacheKey, int taskTimeoutInSeconds) {
         LockEntry myLock = createLockEntry();
 
-        LockEntry existingLock = (LockEntry) crossDCAwareCacheFactory.getCache().putIfAbsent(cacheKey, myLock, taskTimeoutInSeconds, TimeUnit.SECONDS);
+        LockEntry existingLock = InfinispanClusterProviderFactory.putIfAbsentWithRetries(crossDCAwareCacheFactory, cacheKey, myLock, taskTimeoutInSeconds);
         if (existingLock != null) {
             if (logger.isTraceEnabled()) {
                 logger.tracef("Task %s in progress already by node %s. Ignoring task.", cacheKey, existingLock.getNode());
@@ -156,22 +157,15 @@ public class InfinispanClusterProvider implements ClusterProvider {
 
 
     private void removeFromCache(String cacheKey) {
-        // 3 attempts to send the message (it may fail if some node fails in the meantime)
+        // More attempts to send the message (it may fail if some node fails in the meantime)
-        int retry = 3;
+        Retry.executeWithBackoff((int iteration) -> {
-        while (true) {
+
-            try {
             crossDCAwareCacheFactory.getCache().remove(cacheKey);
             if (logger.isTraceEnabled()) {
                 logger.tracef("Task %s removed from the cache", cacheKey);
             }
-                return;
+
-            } catch (RuntimeException e) {
+        }, 10, 10);
-                retry--;
-                if (retry == 0) {
-                    throw e;
-                }
-            }
-        }
     }
 
 }

model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanClusterProviderFactory.java

@@ -18,6 +18,7 @@
 package org.keycloak.cluster.infinispan;
 
 import org.infinispan.Cache;
+import org.infinispan.client.hotrod.exceptions.HotRodClientException;
 import org.infinispan.manager.EmbeddedCacheManager;
 import org.infinispan.notifications.Listener;
 import org.infinispan.notifications.cachemanagerlistener.annotation.ViewChanged;
@@ -29,6 +30,7 @@ import org.jboss.logging.Logger;
 import org.keycloak.Config;
 import org.keycloak.cluster.ClusterProvider;
 import org.keycloak.cluster.ClusterProviderFactory;
+import org.keycloak.common.util.Retry;
 import org.keycloak.common.util.Time;
 import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
 import org.keycloak.models.KeycloakSession;
@@ -42,6 +44,8 @@ import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
@@ -111,7 +115,7 @@ public class InfinispanClusterProviderFactory implements ClusterProviderFactory
             // clusterStartTime not yet initialized. Let's try to put our startupTime
             int serverStartTime = (int) (session.getKeycloakSessionFactory().getServerStartupTimestamp() / 1000);
 
-            existingClusterStartTime = (Integer) crossDCAwareCacheFactory.getCache().putIfAbsent(InfinispanClusterProvider.CLUSTER_STARTUP_TIME_KEY, serverStartTime);
+            existingClusterStartTime = putIfAbsentWithRetries(crossDCAwareCacheFactory, InfinispanClusterProvider.CLUSTER_STARTUP_TIME_KEY, serverStartTime, -1);
             if (existingClusterStartTime == null) {
                 logger.debugf("Initialized cluster startup time to %s", Time.toDate(serverStartTime).toString());
                 return serverStartTime;
@@ -123,6 +127,35 @@ public class InfinispanClusterProviderFactory implements ClusterProviderFactory
     }
 
 
+    // Will retry few times for the case when backup site not available in cross-dc environment.
+    // The site might be taken offline automatically if "take-offline" properly configured
+    static <V extends Serializable> V putIfAbsentWithRetries(CrossDCAwareCacheFactory crossDCAwareCacheFactory, String key, V value, int taskTimeoutInSeconds) {
+        AtomicReference<V> resultRef = new AtomicReference<>();
+
+        Retry.executeWithBackoff((int iteration) -> {
+
+            try {
+                V result;
+                if (taskTimeoutInSeconds > 0) {
+                    result = (V) crossDCAwareCacheFactory.getCache().putIfAbsent(key, value);
+                } else {
+                    result = (V) crossDCAwareCacheFactory.getCache().putIfAbsent(key, value, taskTimeoutInSeconds, TimeUnit.SECONDS);
+                }
+                resultRef.set(result);
+
+            } catch (HotRodClientException re) {
+                logger.warnf(re, "Failed to write key '%s' and value '%s' in iteration '%d' . Retrying", key, value, iteration);
+
+                // Rethrow the exception. Retry will take care of handle the exception and eventually retry the operation.
+                throw re;
+            }
+
+        }, 10, 10);
+
+        return resultRef.get();
+    }
+
+
 
     @Override
     public void init(Config.Scope config) {

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProviderFactory.java

@@ -276,7 +276,7 @@ public class InfinispanUserSessionProviderFactory implements UserSessionProvider
 
 
     private void loadSessionsFromRemoteCache(final KeycloakSessionFactory sessionFactory, String cacheName, final int sessionsPerSegment, final int maxErrors) {
-        log.debugf("Check pre-loading userSessions from remote cache '%s'", cacheName);
+        log.debugf("Check pre-loading sessions from remote cache '%s'", cacheName);
 
         KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
 
@@ -293,7 +293,7 @@ public class InfinispanUserSessionProviderFactory implements UserSessionProvider
 
         });
 
-        log.debugf("Pre-loading userSessions from remote cache '%s' finished", cacheName);
+        log.debugf("Pre-loading sessions from remote cache '%s' finished", cacheName);
     }
 
 

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/initializer/OfflinePersistentUserSessionLoader.java

@@ -18,15 +18,18 @@
 package org.keycloak.models.sessions.infinispan.initializer;
 
 import org.infinispan.Cache;
+import org.infinispan.client.hotrod.exceptions.HotRodClientException;
 import org.infinispan.context.Flag;
 import org.jboss.logging.Logger;
 import org.keycloak.cluster.ClusterProvider;
+import org.keycloak.common.util.Retry;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.UserSessionModel;
 import org.keycloak.models.session.UserSessionPersisterProvider;
 
 import java.io.Serializable;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 
 /**
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
@@ -101,11 +104,25 @@ public class OfflinePersistentUserSessionLoader implements SessionLoader, Serial
     public void afterAllSessionsLoaded(BaseCacheInitializer initializer) {
         Cache<String, Serializable> workCache = initializer.getWorkCache();
 
+        // Will retry few times for the case when backup site not available in cross-dc environment.
+        // The site might be taken offline automatically if "take-offline" properly configured
+        Retry.executeWithBackoff((int iteration) -> {
+
+            try {
                 // Cross-DC aware flag
                 workCache
                         .getAdvancedCache().withFlags(Flag.SKIP_REMOTE_LOOKUP)
                         .put(PERSISTENT_SESSIONS_LOADED, true);
 
+            } catch (HotRodClientException re) {
+                log.warnf(re, "Failed to write flag PERSISTENT_SESSIONS_LOADED in iteration '%d' . Retrying", iteration);
+
+                // Rethrow the exception. Retry will take care of handle the exception and eventually retry the operation.
+                throw re;
+            }
+
+        }, 10, 10);
+
         // Just local-DC aware flag
         workCache
                 .getAdvancedCache().withFlags(Flag.SKIP_REMOTE_LOOKUP, Flag.SKIP_CACHE_LOAD, Flag.SKIP_CACHE_STORE)

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/RemoteCacheSessionsLoader.java

@@ -142,7 +142,8 @@ public class RemoteCacheSessionsLoader implements SessionLoader {
                 .getAdvancedCache().withFlags(Flag.SKIP_CACHE_LOAD, Flag.SKIP_CACHE_STORE)
                 .get(OfflinePersistentUserSessionLoader.PERSISTENT_SESSIONS_LOADED_IN_CURRENT_DC);
 
-        if (cacheName.equals(InfinispanConnectionProvider.OFFLINE_USER_SESSION_CACHE_NAME) && sessionsLoaded != null && sessionsLoaded) {
+        if ((cacheName.equals(InfinispanConnectionProvider.OFFLINE_USER_SESSION_CACHE_NAME) || (cacheName.equals(InfinispanConnectionProvider.OFFLINE_CLIENT_SESSION_CACHE_NAME)))
+                && sessionsLoaded != null && sessionsLoaded) {
             log.debugf("Sessions already loaded in current DC. Skip sessions loading from remote cache '%s'", cacheName);
             return true;
         } else {

model/infinispan/src/test/java/org/keycloak/cluster/infinispan/ConcurrencyJDGOfflineBackupsTest.java

@@ -0,0 +1,114 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.cluster.infinispan;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.infinispan.Cache;
+import org.infinispan.client.hotrod.exceptions.HotRodClientException;
+import org.infinispan.context.Flag;
+import org.infinispan.manager.EmbeddedCacheManager;
+import org.infinispan.persistence.remote.configuration.RemoteStoreConfigurationBuilder;
+import org.jboss.logging.Logger;
+import org.keycloak.common.util.Time;
+import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
+import org.keycloak.models.sessions.infinispan.changes.SessionEntityWrapper;
+import org.keycloak.models.sessions.infinispan.entities.UserSessionEntity;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class ConcurrencyJDGOfflineBackupsTest {
+
+    protected static final Logger logger = Logger.getLogger(ConcurrencyJDGOfflineBackupsTest.class);
+
+    public static void main(String[] args) throws Exception {
+
+        Cache<String, SessionEntityWrapper<UserSessionEntity>> cache1 = createManager(1).getCache(InfinispanConnectionProvider.USER_SESSION_CACHE_NAME);
+
+        try {
+            // Create initial item
+            UserSessionEntity session = new UserSessionEntity();
+            session.setId("123");
+            session.setRealmId("foo");
+            session.setBrokerSessionId("!23123123");
+            session.setBrokerUserId(null);
+            session.setUser("foo");
+            session.setLoginUsername("foo");
+            session.setIpAddress("123.44.143.178");
+            session.setStarted(Time.currentTime());
+            session.setLastSessionRefresh(Time.currentTime());
+
+//        AuthenticatedClientSessionEntity clientSession = new AuthenticatedClientSessionEntity();
+//        clientSession.setAuthMethod("saml");
+//        clientSession.setAction("something");
+//        clientSession.setTimestamp(1234);
+//        clientSession.setProtocolMappers(new HashSet<>(Arrays.asList("mapper1", "mapper2")));
+//        clientSession.setRoles(new HashSet<>(Arrays.asList("role1", "role2")));
+//        session.getAuthenticatedClientSessions().put(CLIENT_1_UUID.toString(), clientSession.getId());
+
+            SessionEntityWrapper<UserSessionEntity> wrappedSession = new SessionEntityWrapper<>(session);
+
+            // Some dummy testing of remoteStore behaviour
+            logger.info("Before put");
+
+
+            AtomicInteger successCount = new AtomicInteger(0);
+            AtomicInteger errorsCount = new AtomicInteger(0);
+            for (int i=0 ; i<100 ; i++) {
+                try {
+                    cache1
+                            .getAdvancedCache().withFlags(Flag.CACHE_MODE_LOCAL) // will still invoke remoteStore . Just doesn't propagate to cluster
+                            .put("123", wrappedSession);
+                    successCount.incrementAndGet();
+                    Thread.sleep(1000);
+                    logger.infof("Success in the iteration: %d", i);
+                } catch (HotRodClientException hrce) {
+                    logger.errorf("Failed to put the item in the iteration: %d ", i);
+                    errorsCount.incrementAndGet();
+                }
+            }
+
+            logger.infof("SuccessCount: %d, ErrorsCount: %d", successCount.get(), errorsCount.get());
+
+//            logger.info("After put");
+//
+//            cache1.replace("123", wrappedSession);
+//
+//            logger.info("After replace");
+//
+//            cache1.get("123");
+//
+//            logger.info("After cache1.get");
+
+//        cache2.get("123");
+//
+//        logger.info("After cache2.get");
+
+        } finally {
+            // Finish JVM
+            cache1.getCacheManager().stop();
+        }
+
+    }
+
+    private static EmbeddedCacheManager createManager(int threadId) {
+        return new TestCacheManagerFactory().createManager(threadId, InfinispanConnectionProvider.USER_SESSION_CACHE_NAME, RemoteStoreConfigurationBuilder.class);
+    }
+
+}

testsuite/integration-arquillian/HOW-TO-RUN.md

@@ -458,114 +458,115 @@ The cross DC requires setting a profile specifying used cache server by specifyi
 
 #### Run Cross-DC Tests from Maven
 
-a) First compile the Infinispan/JDG test server via the following command:
+a) Prepare the environment. Compile the infinispan server and eventually Keycloak on JBoss server.
+
+a1) If you want to use **Undertow** based Keycloak container, you just need to download and prepare the 
+Infinispan/JDG test server via the following command:
 
   `mvn -Pcache-server-infinispan,auth-servers-crossdc-undertow -f testsuite/integration-arquillian -DskipTests clean install`
 
-or
+*note: 'cache-server-infinispan' can be replaced by 'cache-server-jdg'*
 
-  `mvn -Pcache-server-jdg,auth-servers-crossdc-undertow -f testsuite/integration-arquillian -DskipTests clean install`
+a2) If you want to use **JBoss-based** Keycloak backend containers instead of containers on Embedded Undertow,
+ you need to prepare both the Infinispan/JDG test server and the Keycloak server on Wildfly/EAP. Run following command:
 
-b) Then in case you want to use **JBoss-based** Keycloak backend containers instead of containers on Embedded Undertow run following command:
+  `mvn -Pcache-server-infinispan,auth-servers-crossdc-jboss,auth-server-wildfly -f testsuite/integration-arquillian -DskipTests clean install`
 
-    `mvn -Pauth-servers-crossdc-jboss,auth-server-wildfly -f testsuite/integration-arquillian -DskipTests clean install`
+*note: 'cache-server-infinispan' can be replaced by 'cache-server-jdg'*
 
 *note: 'auth-server-wildfly' can be replaced by 'auth-server-eap'*
 
-By default JBoss-based containers use in-memory h2 database. It can be configured to use real DB, e.g. with following command:
+By default JBoss-based containers use TCP-based h2 database. It can be configured to use real DB, e.g. with following command:
 
-  `mvn -Pauth-servers-crossdc-jboss,auth-server-wildfly,jpa -f testsuite/integration-arquillian -DskipTests clean install -Djdbc.mvn.groupId=org.mariadb.jdbc -Djdbc.mvn.artifactId=mariadb-java-client -Djdbc.mvn.version=2.0.3 -Dkeycloak.connectionsJpa.url=jdbc:mariadb://localhost:3306/keycloak -Dkeycloak.connectionsJpa.password=keycloak -Dkeycloak.connectionsJpa.user=keycloak`
+  `mvn -Pcache-server-infinispan,auth-servers-crossdc-jboss,auth-server-wildfly,jpa -f testsuite/integration-arquillian -DskipTests clean install -Djdbc.mvn.groupId=org.mariadb.jdbc -Djdbc.mvn.artifactId=mariadb-java-client -Djdbc.mvn.version=2.0.3 -Dkeycloak.connectionsJpa.url=jdbc:mariadb://localhost:3306/keycloak -Dkeycloak.connectionsJpa.password=keycloak -Dkeycloak.connectionsJpa.user=keycloak`
 
-c1) Then you can run the tests using the following command (adjust the test specification according to your needs) for Keycloak backend containers on **Undertow**:
+b1) For **Undertow** Keycloak backend containers, you can run the tests using the following command (adjust the test specification according to your needs):
 
   `mvn -Pcache-server-infinispan,auth-servers-crossdc-undertow -Dtest=*.crossdc.* -pl testsuite/integration-arquillian/tests/base clean install`
 
-or
+*note: 'cache-server-infinispan' can be replaced by 'cache-server-jdg'*
 
-  `mvn -Pcache-server-jdg,auth-servers-crossdc-undertow -Dtest=*.crossdc.* -pl testsuite/integration-arquillian/tests/base clean install`
+*note: It can be useful to add additional system property to enable logging:*
   
-c2) For **JBoss-based** Keycloak backend containers:
+  `-Dkeycloak.infinispan.logging.level=debug`
+
+b2) For **JBoss-based** Keycloak backend containers, you can run the tests like this:
 
   `mvn -Pcache-server-infinispan,auth-servers-crossdc-jboss,auth-server-wildfly -Dtest=*.crossdc.* -pl testsuite/integration-arquillian/tests/base clean install`
 
-or
+*note: 'cache-server-infinispan' can be replaced by 'cache-server-jdg'*
-
-  `mvn -Pcache-server-jdg,auth-servers-crossdc-jboss,auth-server-wildfly -Dtest=*.crossdc.* -pl testsuite/integration-arquillian/tests/base clean install`
 
 *note: 'auth-server-wildfly can be replaced by auth-server-eap'*
 
-**note**
+**note**:
-Previous commands can be "squashed" into one. E.g.:
+For **JBoss-based** Keycloak backend containers on real DB, the previous commands from (a2) and (b2) can be "squashed" into one. E.g.:
 
   `mvn -f testsuite/integration-arquillian clean install -Dtest=*.crossdc.* -Djdbc.mvn.groupId=org.mariadb.jdbc -Djdbc.mvn.artifactId=mariadb-java-client -Djdbc.mvn.version=2.0.3 -Dkeycloak.connectionsJpa.url=jdbc:mariadb://localhost:3306/keycloak -Dkeycloak.connectionsJpa.password=keycloak -Dkeycloak.connectionsJpa.user=keycloak -Pcache-server-infinispan,auth-servers-crossdc-jboss,auth-server-wildfly,jpa clean install`
 
-It can be useful to add additional system property to enable logging:
     
-    -Dkeycloak.infinispan.logging.level=debug
+#### Run "Manual" Cross-DC Tests from Maven
     
-**Tests from package "manual"** uses manual lifecycle for all servers, so needs to be executed manually. Also needs to be executed with real DB like MySQL. You can run them with:
+Tests from package "manual" uses manual lifecycle for all servers, so needs to be executed manually. 
 
-    mvn -Pcache-server-infinispan -Dtest=*.crossdc.manual.* -Dmanual.mode=true \
+First prepare the environment and do the step (a) from previous paragraph. 
-    -Dkeycloak.connectionsJpa.url.crossdc=jdbc:mysql://localhost/keycloak -Dkeycloak.connectionsJpa.driver.crossdc=com.mysql.jdbc.Driver \
-    -Dkeycloak.connectionsJpa.user=keycloak -Dkeycloak.connectionsJpa.password=keycloak \
-    -pl testsuite/integration-arquillian/tests/base test
 
+c1) For **Undertow** Keycloak backend containers, you can run the test using following command:
 
+  `mvn -Pcache-server-infinispan,auth-servers-crossdc-undertow -Dtest=*.crossdc.manual.* -Dmanual.mode=true -Drun.h2=true -Dkeycloak.connectionsJpa.url.crossdc="jdbc:h2:tcp://localhost:9092/mem:keycloak-dc-shared;DB_CLOSE_DELAY=-1" -pl testsuite/integration-arquillian/tests/base clean install` 
 
+*note: As you can see, there is a need to run TCP-Based H2 for this test. In-memory H2 won't work due the data need 
+to persist the stop of all the Keycloak servers.*
+
+If you want to test with real DB like MySQL, you can run them with:
+
+  `mvn -Pcache-server-infinispan,auth-servers-crossdc-undertow -Dtest=*.crossdc.manual.* -Dmanual.mode=true -Dkeycloak.connectionsJpa.url.crossdc=jdbc:mysql://localhost/keycloak -Dkeycloak.connectionsJpa.driver.crossdc=com.mysql.jdbc.Driver -Dkeycloak.connectionsJpa.user=keycloak -Dkeycloak.connectionsJpa.password=keycloak -pl testsuite/integration-arquillian/tests/base clean install`
+
+c2) For **JBoss-based** Keycloak backend containers, you can run the tests like this:
+    
+  `mvn -Pcache-server-infinispan,auth-servers-crossdc-jboss,auth-server-wildfly -Dtest=*.crossdc.manual.* -Dmanual.mode=true -pl testsuite/integration-arquillian/tests/base clean install`
+    
+*note: TCP-based H2 is used by default when running cross-dc tests on JBoss-based Keycloak container. 
+So no need to explicitly specify it like in (c1) for undertow.*       
      
   
 
 #### Run Cross-DC Tests from Intellij IDEA
 
-First we will manually download, configure and run infinispan server. Then we can run the tests from IDE against 1 server. It's more effective during
+First we will manually download, configure and run infinispan servers. Then we can run the tests from IDE against the servers. 
-development as there is no need to restart infinispan server(s) among test runs.
+It's more effective during development as there is no need to restart infinispan server(s) among test runs.
 
-1) Download infinispan server 8.2.X from http://infinispan.org/download/
+1) Download infinispan server 8.2.X from http://infinispan.org/download/ and go through the steps 
+from the [../../misc/CrossDataCenter.md](../../misc/CrossDataCenter.md) and the `Infinispan Server Setup` part.
 
-2) Edit `ISPN_SERVER_HOME/standalone/configuration/standalone.xml` and add these local-caches to the section under cache-container `local` :
+Assume you have both Infinispan/JDG servers up and running.
 
-    <cache-container name="local" ...
+**TODO:** Change this once CrossDataCenter.md is removed and converted to the proper docs.
 
-        ...
+2) Setup MySQL database or some other shared database.  
 
-        <local-cache-configuration name="sessions-cfg" start="EAGER" batching="false">
+3) Ensure that `org.wildfly.arquillian:wildfly-arquillian-container-managed` is on the classpath when running test. On Intellij, it can be 
-		    <transaction mode="NON_XA" locking="PESSIMISTIC"/>
+done by going to: `View` -> `Tool Windows` -> `Maven projects`. Then check profile `cache-server-infinispan` and `auth-servers-crossdc-undertow`. 
-		</local-cache-configuration>
+The tests will use this profile when executed.
 
-        <local-cache name="sessions" configuration="sessions-cfg" />	
+4) Run the LoginCrossDCTest (or any other test) with those properties. In shortcut, it's using MySQL database and 
-		<local-cache name="offlineSessions" configuration="sessions-cfg" />	
-		<local-cache name="loginFailures" configuration="sessions-cfg" />	
-		<local-cache name="actionTokens" configuration="sessions-cfg" />	
-		<local-cache name="work" configuration="sessions-cfg" />
-    
-    </cache>
-    
-3) Run the server through `./standalone.sh` 
-       
-4) Setup MySQL database or some other shared database.  
-
-5) Ensure that org.wildfly.arquillian:wildfly-arquillian-container-managed is on the classpath when running test. On Intellij, it can be 
-done by going to: View -> Tool Windows -> Maven projects. Then check profile "cache-server-infinispan". The tests will use this profile when executed.
-
-6) Run the LoginCrossDCTest (or any other test) with those properties. In shortcut, it's using MySQL database, disabled L1 lifespan and 
 connects to the remoteStore provided by infinispan server configured in previous steps:
 
-    -Dauth.server.crossdc=true -Dauth.server.undertow.crossdc=true -Dcache.server.lifecycle.skip=true -Dkeycloak.connectionsJpa.url.crossdc=jdbc:mysql://localhost/keycloak 
+  `-Dauth.server.crossdc=true -Dauth.server.undertow.crossdc=true -Dcache.server.lifecycle.skip=true -Dkeycloak.connectionsInfinispan.clustered=true -Dkeycloak.connectionsJpa.url.crossdc=jdbc:mysql://localhost/keycloak -Dkeycloak.connectionsJpa.driver.crossdc=com.mysql.jdbc.Driver -Dkeycloak.connectionsJpa.user=keycloak -Dkeycloak.connectionsJpa.password=keycloak -Dkeycloak.connectionsInfinispan.clustered=true -Dkeycloak.connectionsInfinispan.remoteStorePort=12232 -Dkeycloak.connectionsInfinispan.remoteStorePort.2=13232 -Dkeycloak.connectionsInfinispan.sessionsOwners=1 -Dsession.cache.owners=1 -Dkeycloak.infinispan.logging.level=debug -Dresources`    
-    -Dkeycloak.connectionsJpa.driver.crossdc=com.mysql.jdbc.Driver -Dkeycloak.connectionsJpa.user=keycloak 
-    -Dkeycloak.connectionsJpa.password=keycloak -Dkeycloak.connectionsInfinispan.clustered=true -Dkeycloak.connectionsInfinispan.l1Lifespan=0 
-    -Dkeycloak.connectionsInfinispan.remoteStorePort=11222 -Dkeycloak.connectionsInfinispan.remoteStorePort.2=11222 -Dkeycloak.connectionsInfinispan.sessionsOwners=1 
-    -Dsession.cache.owners=1 -Dkeycloak.infinispan.logging.level=debug -Dresources
     
-NOTE: Tests from package "manual" (eg. SessionsPreloadCrossDCTest) needs to be executed with managed containers. 
+**NOTE**: Tests from package `manual` (eg. SessionsPreloadCrossDCTest) needs to be executed with managed containers. 
 So skip steps 1,2 and add property `-Dmanual.mode=true` and change "cache.server.lifecycle.skip" to false `-Dcache.server.lifecycle.skip=false` or remove it.    
     
-7) If you want to debug and test manually, the servers are running on these ports (Note that not all backend servers are running by default and some might be also unused by loadbalancer):
+5) If you want to debug or test manually, the servers are running on these ports (Note that not all backend servers are running by default and some might be also unused by loadbalancer):
 
-    Loadbalancer -> "http://localhost:8180/auth"
+* *Loadbalancer* -> "http://localhost:8180/auth"
-    auth-server-undertow-cross-dc-0_1 -> "http://localhost:8101/auth"
+
-    auth-server-undertow-cross-dc-0_2-manual -> "http://localhost:8102/auth"
+* *auth-server-undertow-cross-dc-0_1* -> "http://localhost:8101/auth"
-    auth-server-undertow-cross-dc-1_1 -> "http://localhost:8111/auth"
+
-    auth-server-undertow-cross-dc-1_2-manual -> "http://localhost:8112/auth"
+* *auth-server-undertow-cross-dc-0_2-manual* -> "http://localhost:8102/auth"
+
+* *auth-server-undertow-cross-dc-1_1* -> "http://localhost:8111/auth"
+
+* *auth-server-undertow-cross-dc-1_2-manual* -> "http://localhost:8112/auth"
 
 
 ## Run Docker Authentication test

testsuite/integration-arquillian/servers/cache-server/jboss/common/add-keycloak-caches.xsl

@@ -39,7 +39,9 @@
                 <transaction mode="NON_DURABLE_XA" locking="PESSIMISTIC"/>
                 <locking acquire-timeout="0" />
                 <backups>
-                    <backup site="{$remote.site}" failure-policy="FAIL" strategy="SYNC" enabled="true"/>
+                    <backup site="{$remote.site}" failure-policy="FAIL" strategy="SYNC" enabled="true">
+                        <take-offline min-wait="60000" after-failures="3" />
+                    </backup>
                 </backups>
             </replicated-cache-configuration>
 

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/manual/SessionsPreloadCrossDCTest.java

@@ -52,8 +52,9 @@ public class SessionsPreloadCrossDCTest extends AbstractAdminCrossDCTest {
 
         stopAllCacheServersAndAuthServers();
 
-        // Start DC1 only
+        // Start DC1 and only the cache container from DC2. All Keycloak nodes on DC2 are stopped
         containerController.start(getCacheServer(DC.FIRST).getQualifier());
+        containerController.start(getCacheServer(DC.SECOND).getQualifier());
         startBackendNode(DC.FIRST, 0);
         enableLoadBalancerNode(DC.FIRST, 0);
 
@@ -119,7 +120,6 @@ public class SessionsPreloadCrossDCTest extends AbstractAdminCrossDCTest {
         List<OAuthClient.AccessTokenResponse> tokenResponses = createInitialSessions(false);
 
         // Start 2nd DC.
-        containerController.start(getCacheServer(DC.SECOND).getQualifier());
         startBackendNode(DC.SECOND, 0);
         enableLoadBalancerNode(DC.SECOND, 0);
 
@@ -130,7 +130,7 @@ public class SessionsPreloadCrossDCTest extends AbstractAdminCrossDCTest {
         Assert.assertEquals(sessions01, sessionsBefore + SESSIONS_COUNT);
         Assert.assertEquals(sessions02, sessionsBefore + SESSIONS_COUNT);
 
-        // On DC2 sessions were preloaded from from remoteCache
+        // On DC2 sessions were preloaded from remoteCache
         Assert.assertTrue(getTestingClientForStartedNodeInDc(1).testing().cache(InfinispanConnectionProvider.WORK_CACHE_NAME).contains("distributed::remoteCacheLoad::sessions"));
 
         // Assert refreshing works
@@ -157,13 +157,15 @@ public class SessionsPreloadCrossDCTest extends AbstractAdminCrossDCTest {
         // Stop Everything
         stopAllCacheServersAndAuthServers();
 
-        // Start DC1. Sessions should be preloaded from DB
+        // Start cache containers on both DC1 and DC2
         containerController.start(getCacheServer(DC.FIRST).getQualifier());
+        containerController.start(getCacheServer(DC.SECOND).getQualifier());
+
+        // Start Keycloak on DC1. Sessions should be preloaded from DB
         startBackendNode(DC.FIRST, 0);
         enableLoadBalancerNode(DC.FIRST, 0);
 
-        // Start DC2. Sessions should be preloaded from remoteCache
+        // Start Keycloak on DC2. Sessions should be preloaded from remoteCache
-        containerController.start(getCacheServer(DC.SECOND).getQualifier());
         startBackendNode(DC.SECOND, 0);
         enableLoadBalancerNode(DC.SECOND, 0);
 
@@ -210,7 +212,6 @@ public class SessionsPreloadCrossDCTest extends AbstractAdminCrossDCTest {
         }
 
         // Start 2nd DC.
-        containerController.start(getCacheServer(DC.SECOND).getQualifier());
         startBackendNode(DC.SECOND, 0);
         enableLoadBalancerNode(DC.SECOND, 0);
 

travis-run-tests.sh

@@ -82,5 +82,17 @@ if [ $1 == "crossdc" ]; then
     cd tests/base
     mvn clean test -B -nsu -Pcache-server-infinispan,auth-servers-crossdc-jboss,auth-server-wildfly -Dtest=*.crossdc.**.* 2>&1 |
         java -cp ../../../utils/target/classes org.keycloak.testsuite.LogTrimmer
-    exit ${PIPESTATUS[0]}
+    BASE_TESTS_STATUS=${PIPESTATUS[0]}
+
+    mvn clean test -B -nsu -Pcache-server-infinispan,auth-servers-crossdc-jboss,auth-server-wildfly -Dtest=*.crossdc.manual.* -Dmanual.mode=true 2>&1 |
+        java -cp ../../../utils/target/classes org.keycloak.testsuite.LogTrimmer
+    MANUAL_TESTS_STATUS=${PIPESTATUS[0]}
+
+    echo "BASE_TESTS_STATUS=$BASE_TESTS_STATUS, MANUAL_TESTS_STATUS=$MANUAL_TESTS_STATUS";
+    if [ $BASE_TESTS_STATUS -eq 0 -a $MANUAL_TESTS_STATUS -eq 0 ]; then
+        exit 0;
+    else
+        exit 1;
+    fi;
+
 fi