Merge remote-tracking branch 'upstream/master'

common/src/main/java/org/keycloak/common/Version.java

@@ -17,8 +17,6 @@
 
 package org.keycloak.common;
 
-import org.keycloak.common.util.Time;
-
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Properties;

distribution/demo-dist/src/main/xslt/standalone.xsl

@@ -89,6 +89,7 @@
                 <local-cache name="sessions"/>
                 <local-cache name="offlineSessions"/>
                 <local-cache name="loginFailures"/>
+                <local-cache name="work"/>
                 <local-cache name="realmVersions">
                     <transaction mode="BATCH" locking="PESSIMISTIC"/>
                 </local-cache>

distribution/feature-packs/server-feature-pack/src/main/resources/content/standalone/configuration/keycloak-server.json

@@ -37,7 +37,6 @@
     },
 
     "theme": {
-        "default": "keycloak",
         "staticMaxAge": 2592000,
         "cacheTemplates": true,
         "cacheThemes": true,

distribution/server-overlay/cli/keycloak-install-ha.cli

@@ -7,5 +7,8 @@ embed-server --server-config=standalone-ha.xml
 /subsystem=infinispan/cache-container=keycloak/distributed-cache=sessions:add(mode="SYNC",owners="1")
 /subsystem=infinispan/cache-container=keycloak/distributed-cache=offlineSessions:add(mode="SYNC",owners="1")
 /subsystem=infinispan/cache-container=keycloak/distributed-cache=loginFailures:add(mode="SYNC",owners="1")
+/subsystem=infinispan/cache-container=keycloak/replicated-cache=work:add(mode="SYNC")
+/subsystem=infinispan/cache-container=keycloak/local-cache=realmVersions:add()
+/subsystem=infinispan/cache-container=keycloak/local-cache=realmVersions/transaction=TRANSACTION:add(mode=BATCH,locking=PESSIMISTIC)
 /extension=org.keycloak.keycloak-server-subsystem/:add(module=org.keycloak.keycloak-server-subsystem)
 /subsystem=keycloak-server:add(web-context=auth)

distribution/server-overlay/cli/keycloak-install.cli

@@ -6,5 +6,8 @@ embed-server --server-config=standalone.xml
 /subsystem=infinispan/cache-container=keycloak/local-cache=sessions:add()
 /subsystem=infinispan/cache-container=keycloak/local-cache=offlineSessions:add()
 /subsystem=infinispan/cache-container=keycloak/local-cache=loginFailures:add()
+/subsystem=infinispan/cache-container=keycloak/local-cache=work:add()
+/subsystem=infinispan/cache-container=keycloak/local-cache=realmVersions:add()
+/subsystem=infinispan/cache-container=keycloak/local-cache=realmVersions/transaction=TRANSACTION:add(mode=BATCH,locking=PESSIMISTIC)
 /extension=org.keycloak.keycloak-server-subsystem/:add(module=org.keycloak.keycloak-server-subsystem)
 /subsystem=keycloak-server:add(web-context=auth)

integration/admin-client/src/main/java/org/keycloak/admin/client/resource/RealmResource.java

@@ -18,17 +18,19 @@
 package org.keycloak.admin.client.resource;
 
 import org.jboss.resteasy.annotations.cache.NoCache;
+import org.keycloak.representations.idm.AdminEventRepresentation;
 import org.keycloak.representations.idm.ClientRepresentation;
 import org.keycloak.representations.idm.EventRepresentation;
 import org.keycloak.representations.idm.GroupRepresentation;
+import org.keycloak.representations.idm.PartialImportRepresentation;
+import org.keycloak.representations.idm.RealmEventsConfigRepresentation;
 import org.keycloak.representations.idm.RealmRepresentation;
 
 import javax.ws.rs.*;
 import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
 import java.util.List;
 import java.util.Map;
-import org.keycloak.representations.idm.AdminEventRepresentation;
-import org.keycloak.representations.idm.RealmEventsConfigRepresentation;
 
 /**
  * @author rodrigo.sasaki@icarros.com.br
@@ -142,6 +144,12 @@ public interface RealmResource {
     @Path("clients-initial-access")
     ClientInitialAccessResource clientInitialAccess();
 
+    @Path("partialImport")
+    @POST
+    @Consumes(MediaType.APPLICATION_JSON)
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response partialImport(PartialImportRepresentation rep);
+    
     @Path("authentication")
     @Consumes(MediaType.APPLICATION_JSON)
     AuthenticationManagementResource flows();

integration/admin-client/src/main/java/org/keycloak/admin/client/resource/UsersResource.java

@@ -37,9 +37,9 @@ public interface UsersResource {
     List<UserRepresentation> search(@QueryParam("username") String username,
                                            @QueryParam("firstName") String firstName,
                                            @QueryParam("lastName") String lastName,
-                                           @QueryParam("email") String email,
-                                           @QueryParam("first") Integer firstResult,
-                                           @QueryParam("max") Integer maxResults);
+                                             @QueryParam("email") String email,
+                                             @QueryParam("first") Integer firstResult,
+                                             @QueryParam("max") Integer maxResults);
 
     @GET
     @Produces(MediaType.APPLICATION_JSON)

integration/client-registration-cli/pom.xml

@@ -19,9 +19,9 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
     <parent>
-        <artifactId>keycloak-client-registration-parent</artifactId>
+        <artifactId>keycloak-integration-parent</artifactId>
         <groupId>org.keycloak</groupId>
-        <version>1.7.0.Final-SNAPSHOT</version>
+        <version>1.9.0.Final-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

misc/Testsuite.md

@@ -143,6 +143,61 @@ kinit hnelson@KEYCLOAK.ORG
                         
 and provide password `secret`
 
-Now when you access `http://localhost:8081/auth/realms/master/account` you should be logged in automatically as user `hnelson` . 
+Now when you access `http://localhost:8081/auth/realms/master/account` you should be logged in automatically as user `hnelson` .
+ 
+
+Create many users or offline sessions
+-------------------------------------
+Run testsuite with the command like this:
+
+```
+mvn exec:java -Pkeycloak-server -DstartTestsuiteCLI
+```
+
+Alternatively if you want to use your MySQL database use the command like this (replace properties values according your DB connection):
+
+```
+mvn exec:java -Pkeycloak-server -Dkeycloak.connectionsJpa.url=jdbc:mysql://localhost/keycloak -Dkeycloak.connectionsJpa.driver=com.mysql.jdbc.Driver -Dkeycloak.connectionsJpa.user=keycloak -Dkeycloak.connectionsJpa.password=keycloak -DstartTestsuiteCLI
+```
+
+Then once CLI is started, you can use command `help` to see all the available commands. 
+
+### Creating many users
+
+For create many users you can use command `createUsers` 
+For example this will create 500 users `test0, test1, test2, ... , test499` in realm `demo` and each 100 users in separate transaction. All users will be granted realm roles `user` and `admin` :
+
+```
+createUsers test test demo 0 500 100 user,admin
+```
+
+Check count of users:
+
+```
+getUsersCount demo
+```
+
+Check if concrete user was really created:
+
+```
+getUser demo test499
+```
+
+### Creating many offline sessions
+
+For create many offline sessions you can use command `persistSessions` . For example create 50000 sessions (each 500 in separate transaction) with command:
+
+```
+persistSessions 50000 500
+```
+
+Once users or sessions are created, you can restart to ensure the startup import of offline sessions will be triggered and you can see impact on startup time. After restart you can use command:
+
+```
+size
+```
+
+to doublecheck total count of sessions in infinispan (it will be 2 times as there is also 1 client session per each user session created)
+
 
 

model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanClusterProvider.java

@@ -0,0 +1,194 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.cluster.infinispan;
+
+import java.io.Serializable;
+import java.util.concurrent.Callable;
+
+import org.infinispan.Cache;
+import org.infinispan.context.Flag;
+import org.infinispan.lifecycle.ComponentStatus;
+import org.infinispan.remoting.transport.Transport;
+import org.jboss.logging.Logger;
+import org.keycloak.cluster.ClusterEvent;
+import org.keycloak.cluster.ClusterListener;
+import org.keycloak.cluster.ClusterProvider;
+import org.keycloak.cluster.ExecutionResult;
+import org.keycloak.common.util.Time;
+import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
+import org.keycloak.models.KeycloakSession;
+
+/**
+ *
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class InfinispanClusterProvider implements ClusterProvider {
+
+    protected static final Logger logger = Logger.getLogger(InfinispanClusterProvider.class);
+
+    public static final String CLUSTER_STARTUP_TIME_KEY = "cluster-start-time";
+    private static final String TASK_KEY_PREFIX = "task::";
+
+    private final InfinispanClusterProviderFactory factory;
+    private final KeycloakSession session;
+    private final Cache<String, Serializable> cache;
+
+    public InfinispanClusterProvider(InfinispanClusterProviderFactory factory, KeycloakSession session, Cache<String, Serializable> cache) {
+        this.factory = factory;
+        this.session = session;
+        this.cache = cache;
+    }
+
+
+    @Override
+    public int getClusterStartupTime() {
+        Integer existingClusterStartTime = (Integer) cache.get(InfinispanClusterProvider.CLUSTER_STARTUP_TIME_KEY);
+        if (existingClusterStartTime != null) {
+            return existingClusterStartTime;
+        } else {
+            // clusterStartTime not yet initialized. Let's try to put our startupTime
+            int serverStartTime = (int) (session.getKeycloakSessionFactory().getServerStartupTimestamp() / 1000);
+
+            existingClusterStartTime = (Integer) cache.putIfAbsent(InfinispanClusterProvider.CLUSTER_STARTUP_TIME_KEY, serverStartTime);
+            if (existingClusterStartTime == null) {
+                logger.debugf("Initialized cluster startup time to %s", Time.toDate(serverStartTime).toString());
+                return serverStartTime;
+            } else {
+                return existingClusterStartTime;
+            }
+        }
+    }
+
+
+    @Override
+    public void close() {
+    }
+
+
+    @Override
+    public <T> ExecutionResult<T> executeIfNotExecuted(String taskKey, int taskTimeoutInSeconds, Callable<T> task) {
+        String cacheKey = TASK_KEY_PREFIX + taskKey;
+        boolean locked = tryLock(cacheKey, taskTimeoutInSeconds);
+        if (locked) {
+            try {
+                try {
+                    T result = task.call();
+                    return ExecutionResult.executed(result);
+                } catch (RuntimeException re) {
+                    throw re;
+                } catch (Exception e) {
+                    throw new RuntimeException("Unexpected exception when executed task " + taskKey, e);
+                }
+            } finally {
+                removeFromCache(cacheKey);
+            }
+        } else {
+            return ExecutionResult.notExecuted();
+        }
+    }
+
+
+    @Override
+    public void registerListener(String taskKey, ClusterListener task) {
+        factory.registerListener(taskKey, task);
+    }
+
+
+    @Override
+    public void notify(String taskKey, ClusterEvent event) {
+        // Put the value to the cache to notify listeners on all the nodes
+        cache.put(taskKey, event);
+    }
+
+
+    private String getCurrentNode(Cache<String, Serializable> cache) {
+        Transport transport = cache.getCacheManager().getTransport();
+        return transport==null ? "local" : transport.getAddress().toString();
+    }
+
+
+    private LockEntry createLockEntry(Cache<String, Serializable> cache) {
+        LockEntry lock = new LockEntry();
+        lock.setNode(getCurrentNode(cache));
+        lock.setTimestamp(Time.currentTime());
+        return lock;
+    }
+
+
+    private boolean tryLock(String cacheKey, int taskTimeoutInSeconds) {
+        LockEntry myLock = createLockEntry(cache);
+
+        LockEntry existingLock = (LockEntry) cache.putIfAbsent(cacheKey, myLock);
+        if (existingLock != null) {
+            // Task likely already in progress. Check if timestamp is not outdated
+            int thatTime = existingLock.getTimestamp();
+            int currentTime = Time.currentTime();
+            if (thatTime + taskTimeoutInSeconds < currentTime) {
+                if (logger.isTraceEnabled()) {
+                    logger.tracef("Task %s outdated when in progress by node %s. Will try to replace task with our node %s", cacheKey, existingLock.getNode(), myLock.getNode());
+                }
+                boolean replaced = cache.replace(cacheKey, existingLock, myLock);
+                if (!replaced) {
+                    if (logger.isTraceEnabled()) {
+                        logger.tracef("Failed to replace the task %s. Other thread replaced in the meantime. Ignoring task.", cacheKey);
+                    }
+                }
+                return replaced;
+            } else {
+                if (logger.isTraceEnabled()) {
+                    logger.tracef("Task %s in progress already by node %s. Ignoring task.", cacheKey, existingLock.getNode());
+                }
+                return false;
+            }
+        } else {
+            if (logger.isTraceEnabled()) {
+                logger.tracef("Successfully acquired lock for task %s. Our node is %s", cacheKey, myLock.getNode());
+            }
+            return true;
+        }
+    }
+
+
+    private void removeFromCache(String cacheKey) {
+        // 3 attempts to send the message (it may fail if some node fails in the meantime)
+        int retry = 3;
+        while (true) {
+            try {
+                cache.getAdvancedCache()
+                        .withFlags(Flag.IGNORE_RETURN_VALUES, Flag.FORCE_SYNCHRONOUS)
+                        .remove(cacheKey);
+                if (logger.isTraceEnabled()) {
+                    logger.tracef("Task %s removed from the cache", cacheKey);
+                }
+                return;
+            } catch (RuntimeException e) {
+                ComponentStatus status = cache.getStatus();
+                if (status.isStopping() || status.isTerminated()) {
+                    logger.warnf("Failed to remove task %s from the cache. Cache is already terminating", cacheKey);
+                    logger.debug(e.getMessage(), e);
+                    return;
+                }
+                retry--;
+                if (retry == 0) {
+                    throw e;
+                }
+            }
+        }
+    }
+
+}

model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanClusterProviderFactory.java

@@ -0,0 +1,201 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.cluster.infinispan;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+import org.infinispan.Cache;
+import org.infinispan.manager.EmbeddedCacheManager;
+import org.infinispan.notifications.Listener;
+import org.infinispan.notifications.cachelistener.annotation.CacheEntryCreated;
+import org.infinispan.notifications.cachelistener.annotation.CacheEntryModified;
+import org.infinispan.notifications.cachelistener.event.CacheEntryCreatedEvent;
+import org.infinispan.notifications.cachelistener.event.CacheEntryModifiedEvent;
+import org.infinispan.notifications.cachemanagerlistener.annotation.ViewChanged;
+import org.infinispan.notifications.cachemanagerlistener.event.ViewChangedEvent;
+import org.infinispan.remoting.transport.Address;
+import org.infinispan.remoting.transport.Transport;
+import org.jboss.logging.Logger;
+import org.keycloak.Config;
+import org.keycloak.cluster.ClusterEvent;
+import org.keycloak.cluster.ClusterListener;
+import org.keycloak.cluster.ClusterProvider;
+import org.keycloak.cluster.ClusterProviderFactory;
+import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.KeycloakSessionFactory;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class InfinispanClusterProviderFactory implements ClusterProviderFactory {
+
+    public static final String PROVIDER_ID = "infinispan";
+
+    protected static final Logger logger = Logger.getLogger(InfinispanClusterProviderFactory.class);
+
+    private volatile Cache<String, Serializable> workCache;
+
+    private Map<String, ClusterListener> listeners = new HashMap<>();
+
+    @Override
+    public ClusterProvider create(KeycloakSession session) {
+        lazyInit(session);
+        return new InfinispanClusterProvider(this, session, workCache);
+    }
+
+    private void lazyInit(KeycloakSession session) {
+        if (workCache == null) {
+            synchronized (this) {
+                if (workCache == null) {
+                    workCache = session.getProvider(InfinispanConnectionProvider.class).getCache(InfinispanConnectionProvider.WORK_CACHE_NAME);
+                    workCache.getCacheManager().addListener(new ViewChangeListener());
+                    workCache.addListener(new CacheEntryListener());
+                }
+            }
+        }
+    }
+
+    @Override
+    public void init(Config.Scope config) {
+    }
+
+    @Override
+    public void postInit(KeycloakSessionFactory factory) {
+    }
+
+
+    @Override
+    public void close() {
+
+    }
+
+    @Override
+    public String getId() {
+        return PROVIDER_ID;
+    }
+
+
+    @Listener
+    public class ViewChangeListener {
+
+        @ViewChanged
+        public void viewChanged(ViewChangedEvent event) {
+            EmbeddedCacheManager cacheManager = event.getCacheManager();
+            Transport transport = cacheManager.getTransport();
+
+            // Coordinator makes sure that entries for outdated nodes are cleaned up
+            if (transport != null && transport.isCoordinator()) {
+
+                Set<String> newAddresses = convertAddresses(event.getNewMembers());
+                Set<String> removedNodesAddresses = convertAddresses(event.getOldMembers());
+                removedNodesAddresses.removeAll(newAddresses);
+
+                if (removedNodesAddresses.isEmpty()) {
+                    return;
+                }
+
+                logger.debugf("Nodes %s removed from cluster. Removing tasks locked by this nodes", removedNodesAddresses.toString());
+
+                Cache<String, Serializable> cache = cacheManager.getCache(InfinispanConnectionProvider.WORK_CACHE_NAME);
+
+                Iterator<String> toRemove = cache.entrySet().stream().filter(new Predicate<Map.Entry<String, Serializable>>() {
+
+                    @Override
+                    public boolean test(Map.Entry<String, Serializable> entry) {
+                        if (!(entry.getValue() instanceof LockEntry)) {
+                            return false;
+                        }
+
+                        LockEntry lock = (LockEntry) entry.getValue();
+                        return removedNodesAddresses.contains(lock.getNode());
+                    }
+
+                }).map(new Function<Map.Entry<String, Serializable>, String>() {
+
+                    @Override
+                    public String apply(Map.Entry<String, Serializable> entry) {
+                        return entry.getKey();
+                    }
+
+                }).iterator();
+
+                while (toRemove.hasNext()) {
+                    String rem = toRemove.next();
+                    if (logger.isTraceEnabled()) {
+                        logger.tracef("Removing task %s due it's node left cluster", rem);
+                    }
+                    cache.remove(rem);
+                }
+            }
+        }
+
+        private Set<String> convertAddresses(Collection<Address> addresses) {
+            return addresses.stream().map(new Function<Address, String>() {
+
+                @Override
+                public String apply(Address address) {
+                    return address.toString();
+                }
+
+            }).collect(Collectors.toSet());
+        }
+
+    }
+
+
+    <T> void registerListener(String taskKey, ClusterListener task) {
+        listeners.put(taskKey, task);
+    }
+
+    @Listener
+    public class CacheEntryListener {
+
+        @CacheEntryCreated
+        public void cacheEntryCreated(CacheEntryCreatedEvent<String, Object> event) {
+            if (!event.isPre()) {
+                trigger(event.getKey(), event.getValue());
+            }
+        }
+
+        @CacheEntryModified
+        public void cacheEntryModified(CacheEntryModifiedEvent<String, Object> event) {
+            if (!event.isPre()) {
+                trigger(event.getKey(), event.getValue());
+            }
+        }
+
+        private void trigger(String key, Object value) {
+            ClusterListener task = listeners.get(key);
+            if (task != null) {
+                ClusterEvent event = (ClusterEvent) value;
+                task.run(event);
+            }
+        }
+    }
+
+}

model/infinispan/src/main/java/org/keycloak/cluster/infinispan/LockEntry.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.cluster.infinispan;
+
+import java.io.Serializable;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class LockEntry implements Serializable {
+
+    private String node;
+    private int timestamp;
+
+    public String getNode() {
+        return node;
+    }
+
+    public void setNode(String node) {
+        this.node = node;
+    }
+
+    public int getTimestamp() {
+        return timestamp;
+    }
+
+    public void setTimestamp(int timestamp) {
+        this.timestamp = timestamp;
+    }
+}

model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java

@@ -23,6 +23,8 @@ import org.infinispan.configuration.cache.ConfigurationBuilder;
 import org.infinispan.configuration.global.GlobalConfigurationBuilder;
 import org.infinispan.manager.DefaultCacheManager;
 import org.infinispan.manager.EmbeddedCacheManager;
+import org.infinispan.transaction.LockingMode;
+import org.infinispan.transaction.lookup.DummyTransactionManagerLookup;
 import org.jboss.logging.Logger;
 import org.keycloak.Config;
 import org.keycloak.models.KeycloakSession;
@@ -145,6 +147,13 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
         cacheManager.defineConfiguration(InfinispanConnectionProvider.SESSION_CACHE_NAME, sessionCacheConfiguration);
         cacheManager.defineConfiguration(InfinispanConnectionProvider.OFFLINE_SESSION_CACHE_NAME, sessionCacheConfiguration);
         cacheManager.defineConfiguration(InfinispanConnectionProvider.LOGIN_FAILURE_CACHE_NAME, sessionCacheConfiguration);
+
+        ConfigurationBuilder replicationConfigBuilder = new ConfigurationBuilder();
+        if (clustered) {
+            replicationConfigBuilder.clustering().cacheMode(async ? CacheMode.REPL_ASYNC : CacheMode.REPL_SYNC);
+        }
+        Configuration replicationCacheConfiguration = replicationConfigBuilder.build();
+        cacheManager.defineConfiguration(InfinispanConnectionProvider.WORK_CACHE_NAME, replicationCacheConfiguration);
     }
 
 }

model/infinispan/src/main/java/org/keycloak/connections/infinispan/InfinispanConnectionProvider.java

@@ -30,6 +30,7 @@ public interface InfinispanConnectionProvider extends Provider {
     static final String SESSION_CACHE_NAME = "sessions";
     static final String OFFLINE_SESSION_CACHE_NAME = "offlineSessions";
     static final String LOGIN_FAILURE_CACHE_NAME = "loginFailures";
+    static final String WORK_CACHE_NAME = "work";
 
     <K, V> Cache<K, V> getCache(String name);
 

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProvider.java

@@ -21,15 +21,41 @@ import org.infinispan.Cache;
 import org.infinispan.CacheStream;
 import org.jboss.logging.Logger;
 import org.keycloak.common.util.Time;
-import org.keycloak.models.*;
+import org.keycloak.models.ClientInitialAccessModel;
+import org.keycloak.models.ClientModel;
+import org.keycloak.models.ClientSessionModel;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.KeycloakTransaction;
+import org.keycloak.models.RealmModel;
+import org.keycloak.models.UserModel;
+import org.keycloak.models.UserSessionModel;
+import org.keycloak.models.UserSessionProvider;
+import org.keycloak.models.UsernameLoginFailureModel;
 import org.keycloak.models.session.UserSessionPersisterProvider;
-import org.keycloak.models.sessions.infinispan.entities.*;
-import org.keycloak.models.sessions.infinispan.initializer.TimeAwareInitializerState;
-import org.keycloak.models.sessions.infinispan.stream.*;
+import org.keycloak.models.sessions.infinispan.entities.ClientInitialAccessEntity;
+import org.keycloak.models.sessions.infinispan.entities.ClientSessionEntity;
+import org.keycloak.models.sessions.infinispan.entities.LoginFailureEntity;
+import org.keycloak.models.sessions.infinispan.entities.LoginFailureKey;
+import org.keycloak.models.sessions.infinispan.entities.SessionEntity;
+import org.keycloak.models.sessions.infinispan.entities.UserSessionEntity;
+import org.keycloak.models.sessions.infinispan.stream.ClientInitialAccessPredicate;
+import org.keycloak.models.sessions.infinispan.stream.ClientSessionPredicate;
+import org.keycloak.models.sessions.infinispan.stream.Comparators;
+import org.keycloak.models.sessions.infinispan.stream.Mappers;
+import org.keycloak.models.sessions.infinispan.stream.SessionPredicate;
+import org.keycloak.models.sessions.infinispan.stream.UserLoginFailurePredicate;
+import org.keycloak.models.sessions.infinispan.stream.UserSessionPredicate;
 import org.keycloak.models.utils.KeycloakModelUtils;
 import org.keycloak.models.utils.RealmInfoUtil;
 
-import java.util.*;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.function.Consumer;
 import java.util.function.Predicate;
 import java.util.stream.Stream;
@@ -411,19 +437,6 @@ public class InfinispanUserSessionProvider implements UserSessionProvider {
         loginFailureCache.remove(new LoginFailureKey(realm.getId(), user.getEmail()));
     }
 
-    @Override
-    public int getClusterStartupTime() {
-        TimeAwareInitializerState state = (TimeAwareInitializerState) offlineSessionCache.get(InfinispanUserSessionProviderFactory.SESSION_INITIALIZER_STATE_KEY);
-        int startTime;
-        if (state == null) {
-            log.warn("Cluster startup time not yet available. Fallback to local startup time");
-            startTime = (int)(session.getKeycloakSessionFactory().getServerStartupTimestamp() / 1000);
-        } else {
-            startTime = state.getClusterStartupTime();
-        }
-        return startTime;
-    }
-
     @Override
     public void close() {
     }

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProviderFactory.java

@@ -17,6 +17,8 @@
 
 package org.keycloak.models.sessions.infinispan;
 
+import java.io.Serializable;
+
 import org.infinispan.Cache;
 import org.jboss.logging.Logger;
 import org.keycloak.Config;
@@ -34,9 +36,6 @@ import org.keycloak.provider.ProviderEventListener;
 
 public class InfinispanUserSessionProviderFactory implements UserSessionProviderFactory {
 
-    private static final String STATE_KEY_PREFIX = "initializerState";
-    public static final String SESSION_INITIALIZER_STATE_KEY = STATE_KEY_PREFIX + "::offlineUserSessions";
-
     private static final Logger log = Logger.getLogger(InfinispanUserSessionProviderFactory.class);
 
     private Config.Scope config;
@@ -85,9 +84,9 @@ public class InfinispanUserSessionProviderFactory implements UserSessionProvider
             @Override
             public void run(KeycloakSession session) {
                 InfinispanConnectionProvider connections = session.getProvider(InfinispanConnectionProvider.class);
-                Cache<String, SessionEntity> cache = connections.getCache(InfinispanConnectionProvider.OFFLINE_SESSION_CACHE_NAME);
+                Cache<String, Serializable> cache = connections.getCache(InfinispanConnectionProvider.WORK_CACHE_NAME);
 
-                InfinispanUserSessionInitializer initializer = new InfinispanUserSessionInitializer(sessionFactory, cache, new OfflineUserSessionLoader(), maxErrors, sessionsPerSegment, SESSION_INITIALIZER_STATE_KEY);
+                InfinispanUserSessionInitializer initializer = new InfinispanUserSessionInitializer(sessionFactory, cache, new OfflineUserSessionLoader(), maxErrors, sessionsPerSegment, "offlineUserSessions");
                 initializer.initCache();
                 initializer.loadPersistentSessions();
             }

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/initializer/InfinispanUserSessionInitializer.java

@@ -20,57 +20,55 @@ package org.keycloak.models.sessions.infinispan.initializer;
 import java.io.Serializable;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
 
 import org.infinispan.Cache;
 import org.infinispan.context.Flag;
 import org.infinispan.distexec.DefaultExecutorService;
-import org.infinispan.notifications.Listener;
-import org.infinispan.notifications.cachemanagerlistener.annotation.ViewChanged;
-import org.infinispan.notifications.cachemanagerlistener.event.ViewChangedEvent;
+import org.infinispan.lifecycle.ComponentStatus;
 import org.infinispan.remoting.transport.Transport;
 import org.jboss.logging.Logger;
-import org.keycloak.common.util.Time;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
 import org.keycloak.models.KeycloakSessionTask;
-import org.keycloak.models.sessions.infinispan.entities.SessionEntity;
 import org.keycloak.models.utils.KeycloakModelUtils;
 
 /**
  * Startup initialization for reading persistent userSessions/clientSessions to be filled into infinispan/memory . In cluster,
  * the initialization is distributed among all cluster nodes, so the startup time is even faster
  *
+ * TODO: Move to clusterService. Implementation is already pretty generic and doesn't contain any "userSession" specific stuff. All sessions-specific logic is in the SessionLoader implementation
+ *
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
 public class InfinispanUserSessionInitializer {
 
+    private static final String STATE_KEY_PREFIX = "distributed::";
+
     private static final Logger log = Logger.getLogger(InfinispanUserSessionInitializer.class);
 
     private final KeycloakSessionFactory sessionFactory;
-    private final Cache<String, SessionEntity> cache;
+    private final Cache<String, Serializable> workCache;
     private final SessionLoader sessionLoader;
     private final int maxErrors;
     private final int sessionsPerSegment;
     private final String stateKey;
 
 
-    public InfinispanUserSessionInitializer(KeycloakSessionFactory sessionFactory, Cache<String, SessionEntity> cache, SessionLoader sessionLoader, int maxErrors, int sessionsPerSegment, String stateKey) {
+    public InfinispanUserSessionInitializer(KeycloakSessionFactory sessionFactory, Cache<String, Serializable> workCache, SessionLoader sessionLoader, int maxErrors, int sessionsPerSegment, String stateKeySuffix) {
         this.sessionFactory = sessionFactory;
-        this.cache = cache;
+        this.workCache = workCache;
         this.sessionLoader = sessionLoader;
         this.maxErrors = maxErrors;
         this.sessionsPerSegment = sessionsPerSegment;
-        this.stateKey = stateKey;
+        this.stateKey = STATE_KEY_PREFIX + stateKeySuffix;
     }
 
     public void initCache() {
-        this.cache.getAdvancedCache().getComponentRegistry().registerComponent(sessionFactory, KeycloakSessionFactory.class);
+        this.workCache.getAdvancedCache().getComponentRegistry().registerComponent(sessionFactory, KeycloakSessionFactory.class);
     }
 
 
@@ -94,16 +92,14 @@ public class InfinispanUserSessionInitializer {
 
 
     private boolean isFinished() {
-        InitializerState state = (InitializerState) cache.get(stateKey);
+        InitializerState state = (InitializerState) workCache.get(stateKey);
         return state != null && state.isFinished();
     }
 
 
     private InitializerState getOrCreateInitializerState() {
-        TimeAwareInitializerState state = (TimeAwareInitializerState) cache.get(stateKey);
+        InitializerState state = (InitializerState) workCache.get(stateKey);
         if (state == null) {
-            int startTime = (int)(sessionFactory.getServerStartupTimestamp() / 1000);
-
             final int[] count = new int[1];
 
             // Rather use separate transactions for update and counting
@@ -111,7 +107,7 @@ public class InfinispanUserSessionInitializer {
             KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
                 @Override
                 public void run(KeycloakSession session) {
-                    sessionLoader.init(session, startTime);
+                    sessionLoader.init(session);
                 }
 
             });
@@ -124,9 +120,8 @@ public class InfinispanUserSessionInitializer {
 
             });
 
-            state = new TimeAwareInitializerState();
+            state = new InitializerState();
             state.init(count[0], sessionsPerSegment);
-            state.setClusterStartupTime(startTime);
             saveStateToCache(state);
         }
         return state;
@@ -143,7 +138,7 @@ public class InfinispanUserSessionInitializer {
             public void run() {
 
                 // Save this synchronously to ensure all nodes read correct state
-                InfinispanUserSessionInitializer.this.cache.getAdvancedCache().
+                InfinispanUserSessionInitializer.this.workCache.getAdvancedCache().
                         withFlags(Flag.IGNORE_RETURN_VALUES, Flag.FORCE_SYNCHRONOUS)
                         .put(stateKey, state);
             }
@@ -153,7 +148,7 @@ public class InfinispanUserSessionInitializer {
 
 
     private boolean isCoordinator() {
-        Transport transport = cache.getCacheManager().getTransport();
+        Transport transport = workCache.getCacheManager().getTransport();
         return transport == null || transport.isCoordinator();
     }
 
@@ -166,9 +161,9 @@ public class InfinispanUserSessionInitializer {
         int processors = Runtime.getRuntime().availableProcessors();
 
         ExecutorService localExecutor = Executors.newCachedThreadPool();
-        Transport transport = cache.getCacheManager().getTransport();
+        Transport transport = workCache.getCacheManager().getTransport();
         boolean distributed = transport != null;
-        ExecutorService executorService = distributed ? new DefaultExecutorService(cache, localExecutor) : localExecutor;
+        ExecutorService executorService = distributed ? new DefaultExecutorService(workCache, localExecutor) : localExecutor;
 
         int errors = 0;
 
@@ -190,7 +185,7 @@ public class InfinispanUserSessionInitializer {
                     SessionInitializerWorker worker = new SessionInitializerWorker();
                     worker.setWorkerEnvironment(segment, sessionsPerSegment, sessionLoader);
                     if (!distributed) {
-                        worker.setEnvironment(cache, null);
+                        worker.setEnvironment(workCache, null);
                     }
 
                     Future<WorkerResult> future = executorService.submit(worker);
@@ -242,6 +237,12 @@ public class InfinispanUserSessionInitializer {
                 runnable.run();
                 return;
             } catch (RuntimeException e) {
+                ComponentStatus status = workCache.getStatus();
+                if (status.isStopping() || status.isTerminated()) {
+                    log.warn("Failed to put initializerState to the cache. Cache is already terminating");
+                    log.debug(e.getMessage(), e);
+                    return;
+                }
                 retry--;
                 if (retry == 0) {
                     throw e;

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/initializer/OfflineUserSessionLoader.java

@@ -20,6 +20,7 @@ package org.keycloak.models.sessions.infinispan.initializer;
 import java.util.List;
 
 import org.jboss.logging.Logger;
+import org.keycloak.cluster.ClusterProvider;
 import org.keycloak.models.ClientSessionModel;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.UserSessionModel;
@@ -33,9 +34,12 @@ public class OfflineUserSessionLoader implements SessionLoader {
     private static final Logger log = Logger.getLogger(OfflineUserSessionLoader.class);
 
     @Override
-    public void init(KeycloakSession session, int clusterStartupTime) {
+    public void init(KeycloakSession session) {
         UserSessionPersisterProvider persister = session.getProvider(UserSessionPersisterProvider.class);
 
+        // TODO: check if update of timestamps in persister can be skipped entirely
+        int clusterStartupTime = session.getProvider(ClusterProvider.class).getClusterStartupTime();
+
         log.debugf("Clearing detached sessions from persistent storage and updating timestamps to %d", clusterStartupTime);
 
         persister.clearDetachedUserSessions();

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/initializer/SessionInitializerWorker.java

@@ -32,7 +32,7 @@ import org.keycloak.models.utils.KeycloakModelUtils;
 /**
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
-public class SessionInitializerWorker implements DistributedCallable<String, SessionEntity, InfinispanUserSessionInitializer.WorkerResult>, Serializable {
+public class SessionInitializerWorker implements DistributedCallable<String, Serializable, InfinispanUserSessionInitializer.WorkerResult>, Serializable {
 
     private static final Logger log = Logger.getLogger(SessionInitializerWorker.class);
 
@@ -40,7 +40,7 @@ public class SessionInitializerWorker implements DistributedCallable<String, Ses
     private int sessionsPerSegment;
     private SessionLoader sessionLoader;
 
-    private transient Cache<String, SessionEntity> cache;
+    private transient Cache<String, Serializable> workCache;
 
     public void setWorkerEnvironment(int segment, int sessionsPerSegment, SessionLoader sessionLoader) {
         this.segment = segment;
@@ -49,8 +49,8 @@ public class SessionInitializerWorker implements DistributedCallable<String, Ses
     }
 
     @Override
-    public void setEnvironment(Cache<String, SessionEntity> cache, Set<String> inputKeys) {
-        this.cache = cache;
+    public void setEnvironment(Cache<String, Serializable> workCache, Set<String> inputKeys) {
+        this.workCache = workCache;
     }
 
     @Override
@@ -59,7 +59,7 @@ public class SessionInitializerWorker implements DistributedCallable<String, Ses
             log.tracef("Running computation for segment: %d", segment);
         }
 
-        KeycloakSessionFactory sessionFactory = cache.getAdvancedCache().getComponentRegistry().getComponent(KeycloakSessionFactory.class);
+        KeycloakSessionFactory sessionFactory = workCache.getAdvancedCache().getComponentRegistry().getComponent(KeycloakSessionFactory.class);
         if (sessionFactory == null) {
             log.warnf("KeycloakSessionFactory not yet set in cache. Worker skipped");
             return InfinispanUserSessionInitializer.WorkerResult.create(segment, false);

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/initializer/SessionLoader.java

@@ -26,7 +26,7 @@ import org.keycloak.models.KeycloakSession;
  */
 public interface SessionLoader extends Serializable {
 
-    void init(KeycloakSession session, int clusterStartupTime);
+    void init(KeycloakSession session);
 
     int getSessionsCount(KeycloakSession session);
 

model/infinispan/src/main/resources/META-INF/services/org.keycloak.cluster.ClusterProviderFactory

@@ -0,0 +1,18 @@
+#
+# Copyright 2016 Red Hat, Inc. and/or its affiliates
+# and other contributors as indicated by the @author tags.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.keycloak.cluster.infinispan.InfinispanClusterProviderFactory

model/jpa/src/main/resources/META-INF/jpa-changelog-1.9.0.xml

@@ -73,5 +73,9 @@
         <dropUniqueConstraint tableName="FED_PROVIDERS" constraintName="UK_DCCIRJLIPU1478VQC89DID88C" />
         <dropTable tableName="FED_PROVIDERS" />
 
+        <createIndex indexName="IDX_US_SESS_ID_ON_CL_SESS" tableName="OFFLINE_CLIENT_SESSION">
+            <column name="USER_SESSION_ID" type="VARCHAR(36)"/>
+        </createIndex>
+
     </changeSet>
 </databaseChangeLog>

saml-core/src/main/java/org/keycloak/saml/processing/core/util/XMLSignatureUtil.java

@@ -18,6 +18,7 @@ package org.keycloak.saml.processing.core.util;
 
 import org.keycloak.saml.common.PicketLinkLogger;
 import org.keycloak.saml.common.PicketLinkLoggerFactory;
+import org.keycloak.saml.common.constants.JBossSAMLConstants;
 import org.keycloak.saml.common.constants.JBossSAMLURIConstants;
 import org.keycloak.saml.common.constants.WSTrustConstants;
 import org.keycloak.saml.common.exceptions.ParsingException;
@@ -376,7 +377,19 @@ public class XMLSignatureUtil {
         if (publicKey == null)
             throw logger.nullValueError("Public Key");
 
+        int signedAssertions = 0;
+        String assertionNameSpaceUri = null;
+
         for (int i = 0; i < nl.getLength(); i++) {
+            Node signatureNode = nl.item(i);
+            Node parent = signatureNode.getParentNode();
+            if (parent != null && JBossSAMLConstants.ASSERTION.get().equals(parent.getLocalName())) {
+                ++signedAssertions;
+                if (assertionNameSpaceUri == null) {
+                    assertionNameSpaceUri = parent.getNamespaceURI();
+                }
+            }
+
             DOMValidateContext valContext = new DOMValidateContext(publicKey, nl.item(i));
             XMLSignature signature = fac.unmarshalXMLSignature(valContext);
 
@@ -397,6 +410,16 @@ public class XMLSignatureUtil {
             }
         }
 
+        NodeList assertions = signedDoc.getElementsByTagNameNS(assertionNameSpaceUri, JBossSAMLConstants.ASSERTION.get());
+
+        if (signedAssertions > 0 && assertions != null && assertions.getLength() != signedAssertions) {
+            if (logger.isDebugEnabled()) {
+                logger.debug("SAML Response document may contain malicious assertions. Signature validation will fail.");
+            }
+            // there are unsigned assertions mixed with signed ones
+            return false;
+        }
+
         return true;
     }
 

server-spi/src/main/java/org/keycloak/cluster/ClusterEvent.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.cluster;
+
+import java.io.Serializable;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public interface ClusterEvent extends Serializable {
+}

server-spi/src/main/java/org/keycloak/cluster/ClusterListener.java

@@ -15,20 +15,20 @@
  * limitations under the License.
  */
 
-package org.keycloak.models.sessions.infinispan.initializer;
+package org.keycloak.cluster;
 
 /**
+ * Task to be executed on all cluster nodes once it's notified.
+ *
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
-public class TimeAwareInitializerState extends InitializerState {
+public interface ClusterListener {
 
-    private int clusterStartupTime;
+    /**
+     * Registered task to be executed on all cluster nodes once it's notified from cache.
+     *
+     * @param event value of notification (Object added into the cache)
+     */
+    void run(ClusterEvent event);
 
-    public int getClusterStartupTime() {
-        return clusterStartupTime;
-    }
-
-    public void setClusterStartupTime(int clusterStartupTime) {
-        this.clusterStartupTime = clusterStartupTime;
-    }
 }

server-spi/src/main/java/org/keycloak/cluster/ClusterProvider.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.cluster;
+
+
+import java.util.concurrent.Callable;
+
+import org.keycloak.provider.Provider;
+
+/**
+ * Various utils related to clustering and concurrent tasks on cluster nodes
+ *
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public interface ClusterProvider extends Provider {
+
+    /**
+     * Same value for all cluster nodes. It will use startup time of this server in non-cluster environment.
+     */
+    int getClusterStartupTime();
+
+
+    /**
+     * Execute given task just if it's not already in progress (either on this or any other cluster node).
+     *
+     * @param taskKey
+     * @param taskTimeoutInSeconds timeout for given task. If there is existing task in progress for longer time, it's considered outdated so we will start our task.
+     * @param task
+     * @param <T>
+     * @return result with "executed" flag specifying if execution was executed or ignored.
+     */
+    <T> ExecutionResult<T> executeIfNotExecuted(String taskKey, int taskTimeoutInSeconds, Callable<T> task);
+
+
+    /**
+     * Register task (listener) under given key. When this key will be put to the cache on any cluster node, the task will be executed
+     *
+     * @param taskKey
+     * @param task
+     */
+    void registerListener(String taskKey, ClusterListener task);
+
+
+    /**
+     * Notify registered listeners on all cluster nodes
+     *
+     * @param taskKey
+     * @param event
+     */
+    void notify(String taskKey, ClusterEvent event);
+}

server-spi/src/main/java/org/keycloak/cluster/ClusterProviderFactory.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.cluster;
+
+import org.keycloak.provider.ProviderFactory;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public interface ClusterProviderFactory extends ProviderFactory<ClusterProvider> {
+}

server-spi/src/main/java/org/keycloak/cluster/ClusterSpi.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.cluster;
+
+import org.keycloak.provider.Provider;
+import org.keycloak.provider.ProviderFactory;
+import org.keycloak.provider.Spi;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class ClusterSpi implements Spi {
+
+    @Override
+    public boolean isInternal() {
+        return true;
+    }
+
+    @Override
+    public String getName() {
+        return "cluster";
+    }
+
+    @Override
+    public Class<? extends Provider> getProviderClass() {
+        return ClusterProvider.class;
+    }
+
+    @Override
+    public Class<? extends ProviderFactory> getProviderFactoryClass() {
+        return ClusterProviderFactory.class;
+    }
+}

server-spi/src/main/java/org/keycloak/cluster/ExecutionResult.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.cluster;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class ExecutionResult<T> {
+
+    private final boolean executed;
+    private final T result;
+
+    private ExecutionResult(boolean executed, T result) {
+        this.executed = executed;
+        this.result = result;
+    }
+
+    public static <T> ExecutionResult<T> executed(T result) {
+        return new ExecutionResult<>(true, result);
+    }
+
+    public static <T> ExecutionResult<T> notExecuted() {
+        return new ExecutionResult<>(false, null);
+    }
+
+    public boolean isExecuted() {
+        return executed;
+    }
+
+    public T getResult() {
+        return result;
+    }
+}

server-spi/src/main/java/org/keycloak/models/UserFederationSyncResult.java

@@ -22,11 +22,21 @@ package org.keycloak.models;
  */
 public class UserFederationSyncResult {
 
+    private boolean ignored;
+
     private int added;
     private int updated;
     private int removed;
     private int failed;
 
+    public boolean isIgnored() {
+        return ignored;
+    }
+
+    public void setIgnored(boolean ignored) {
+        this.ignored = ignored;
+    }
+
     public int getAdded() {
         return added;
     }
@@ -83,11 +93,15 @@ public class UserFederationSyncResult {
     }
 
     public String getStatus() {
-        String status = String.format("%d imported users, %d updated users, %d removed users", added, updated, removed);
-        if (failed != 0) {
-            status += String.format(", %d users failed sync! See server log for more details", failed);
+        if (ignored) {
+            return "Synchronization ignored as it's already in progress";
+        } else {
+            String status = String.format("%d imported users, %d updated users, %d removed users", added, updated, removed);
+            if (failed != 0) {
+                status += String.format(", %d users failed sync! See server log for more details", failed);
+            }
+            return status;
         }
-        return status;
     }
 
     @Override
@@ -98,4 +112,10 @@ public class UserFederationSyncResult {
     public static UserFederationSyncResult empty() {
         return new UserFederationSyncResult();
     }
+
+    public static UserFederationSyncResult ignored() {
+        UserFederationSyncResult result = new UserFederationSyncResult();
+        result.setIgnored(true);
+        return result;
+    }
 }

server-spi/src/main/java/org/keycloak/models/UserSessionProvider.java

@@ -82,9 +82,6 @@ public interface UserSessionProvider extends Provider {
     void removeClientInitialAccessModel(RealmModel realm, String id);
     List<ClientInitialAccessModel> listClientInitialAccess(RealmModel realm);
 
-    // Will use startup time of this server in non-cluster environment
-    int getClusterStartupTime();
-
     void close();
 
 }

server-spi/src/main/resources/META-INF/services/org.keycloak.provider.Spi

@@ -49,5 +49,6 @@ org.keycloak.authentication.ClientAuthenticatorSpi
 org.keycloak.authentication.RequiredActionSpi
 org.keycloak.authentication.FormAuthenticatorSpi
 org.keycloak.authentication.FormActionSpi
+org.keycloak.cluster.ClusterSpi
 
 

services/src/main/java/org/keycloak/partialimport/PartialImportResult.java

@@ -26,11 +26,13 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
  */
 public class PartialImportResult {
 
-    private final Action action;
-    private final ResourceType resourceType;
-    private final String resourceName;
-    private final String id;
-    private final Object representation;
+    private Action action;
+    private ResourceType resourceType;
+    private String resourceName;
+    private String id;
+    private Object representation;
+
+    private PartialImportResult() {};
 
     private PartialImportResult(Action action, ResourceType resourceType, String resourceName, String id, Object representation) {
         this.action = action;
@@ -56,18 +58,34 @@ public class PartialImportResult {
         return action;
     }
 
+    public void setAction(Action action) {
+        this.action = action;
+    }
+
     public ResourceType getResourceType() {
         return resourceType;
     }
 
+    public void setResourceType(ResourceType resourceType) {
+        this.resourceType = resourceType;
+    }
+
     public String getResourceName() {
         return resourceName;
     }
 
+    public void setResourceName(String resourceName) {
+        this.resourceName = resourceName;
+    }
+
     public String getId() {
         return id;
     }
 
+    public void setId(String id) {
+        this.id = id;
+    }
+
     @JsonIgnore
     public Object getRepresentation() {
         return representation;

services/src/main/java/org/keycloak/partialimport/PartialImportResults.java

@@ -28,6 +28,14 @@ import java.util.Set;
  */
 public class PartialImportResults {
 
+    // these fields used only for marsalling from JSON with admin client
+    // they are never directly set
+    private int overwritten;
+    private int added;
+    private int skipped;
+
+    private String errorMessage;
+
     private final Set<PartialImportResult> importResults = new HashSet<>();
 
     public void addResult(PartialImportResult result) {
@@ -68,4 +76,13 @@ public class PartialImportResults {
     public Set<PartialImportResult> getResults() {
         return importResults;
     }
+
+    public String getErrorMessage() {
+        return errorMessage;
+    }
+
+    public void setErrorMessage(String errorMessage) {
+        this.errorMessage = errorMessage;
+    }
+
 }

services/src/main/java/org/keycloak/protocol/oidc/TokenManager.java

@@ -17,6 +17,7 @@
 
 package org.keycloak.protocol.oidc;
 
+import org.keycloak.cluster.ClusterProvider;
 import org.keycloak.common.ClientConnection;
 import org.keycloak.OAuth2Constants;
 import org.keycloak.OAuthErrorException;
@@ -193,7 +194,7 @@ public class TokenManager {
         int currentTime = Time.currentTime();
 
         if (realm.isRevokeRefreshToken()) {
-            int clusterStartupTime = session.sessions().getClusterStartupTime();
+            int clusterStartupTime = session.getProvider(ClusterProvider.class).getClusterStartupTime();
 
             if (refreshToken.getIssuedAt() < validation.clientSession.getTimestamp() && (clusterStartupTime != validation.clientSession.getTimestamp())) {
                 throw new OAuthErrorException(OAuthErrorException.INVALID_GRANT, "Stale token");

services/src/main/java/org/keycloak/services/managers/RealmManager.java

@@ -231,7 +231,7 @@ public class RealmManager implements RealmImporter {
             // Remove all periodic syncs for configured federation providers
             UsersSyncManager usersSyncManager = new UsersSyncManager();
             for (final UserFederationProviderModel fedProvider : federationProviders) {
-                usersSyncManager.removePeriodicSyncForProvider(session.getProvider(TimerProvider.class), fedProvider);
+                usersSyncManager.notifyToRefreshPeriodicSync(session, realm, fedProvider, true);
             }
         }
         return removed;
@@ -434,7 +434,7 @@ public class RealmManager implements RealmImporter {
         List<UserFederationProviderModel> federationProviders = realm.getUserFederationProviders();
         UsersSyncManager usersSyncManager = new UsersSyncManager();
         for (final UserFederationProviderModel fedProvider : federationProviders) {
-            usersSyncManager.refreshPeriodicSyncForProvider(session.getKeycloakSessionFactory(), session.getProvider(TimerProvider.class), fedProvider, realm.getId());
+            usersSyncManager.notifyToRefreshPeriodicSync(session, realm, fedProvider, false);
         }
         return realm;
     }

services/src/main/java/org/keycloak/services/managers/UsersSyncManager.java

@@ -16,6 +16,10 @@
  */
 package org.keycloak.services.managers;
 
+import org.keycloak.cluster.ClusterEvent;
+import org.keycloak.cluster.ClusterListener;
+import org.keycloak.cluster.ClusterProvider;
+import org.keycloak.cluster.ExecutionResult;
 import org.keycloak.common.util.Time;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
@@ -30,13 +34,17 @@ import org.keycloak.services.ServicesLogger;
 import org.keycloak.timer.TimerProvider;
 
 
+import java.io.Serializable;
 import java.util.List;
+import java.util.concurrent.Callable;
 
 /**
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
 public class UsersSyncManager {
 
+    private static final String FEDERATION_TASK_KEY = "federation";
+
     protected static final ServicesLogger logger = ServicesLogger.ROOT_LOGGER;
 
     /**
@@ -57,26 +65,106 @@ public class UsersSyncManager {
                         refreshPeriodicSyncForProvider(sessionFactory, timer, fedProvider, realm.getId());
                     }
                 }
+
+                ClusterProvider clusterProvider = session.getProvider(ClusterProvider.class);
+                clusterProvider.registerListener(FEDERATION_TASK_KEY, new UserFederationClusterListener(sessionFactory));
             }
         });
     }
 
-    public UserFederationSyncResult syncAllUsers(final KeycloakSessionFactory sessionFactory, String realmId, final UserFederationProviderModel fedProvider) {
-        final UserFederationProviderFactory fedProviderFactory = (UserFederationProviderFactory) sessionFactory.getProviderFactory(UserFederationProvider.class, fedProvider.getProviderName());
-        updateLastSyncInterval(sessionFactory, fedProvider, realmId);
-        return fedProviderFactory.syncAllUsers(sessionFactory, realmId, fedProvider);
+
+    private class Holder {
+        ExecutionResult<UserFederationSyncResult> result;
     }
 
-    public UserFederationSyncResult syncChangedUsers(final KeycloakSessionFactory sessionFactory, String realmId, final UserFederationProviderModel fedProvider) {
-        final UserFederationProviderFactory fedProviderFactory = (UserFederationProviderFactory) sessionFactory.getProviderFactory(UserFederationProvider.class, fedProvider.getProviderName());
+    public UserFederationSyncResult syncAllUsers(final KeycloakSessionFactory sessionFactory, final String realmId, final UserFederationProviderModel fedProvider) {
+        final Holder holder = new Holder();
 
-        // See when we did last sync.
-        int oldLastSync = fedProvider.getLastSync();
-        updateLastSyncInterval(sessionFactory, fedProvider, realmId);
-        return fedProviderFactory.syncChangedUsers(sessionFactory, realmId, fedProvider, Time.toDate(oldLastSync));
+        // Ensure not executed concurrently on this or any other cluster node
+        KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
+
+            @Override
+            public void run(KeycloakSession session) {
+                ClusterProvider clusterProvider = session.getProvider(ClusterProvider.class);
+                // shared key for "full" and "changed" . Improve if needed
+                String taskKey = fedProvider.getId() + "::sync";
+
+                // 30 seconds minimal timeout for now
+                int timeout = Math.max(30, fedProvider.getFullSyncPeriod());
+                holder.result = clusterProvider.executeIfNotExecuted(taskKey, timeout, new Callable<UserFederationSyncResult>() {
+
+                    @Override
+                    public UserFederationSyncResult call() throws Exception {
+                        final UserFederationProviderFactory fedProviderFactory = (UserFederationProviderFactory) sessionFactory.getProviderFactory(UserFederationProvider.class, fedProvider.getProviderName());
+                        updateLastSyncInterval(sessionFactory, fedProvider, realmId);
+                        return fedProviderFactory.syncAllUsers(sessionFactory, realmId, fedProvider);
+                    }
+
+                });
+            }
+
+        });
+
+        if (holder.result == null || !holder.result.isExecuted()) {
+            logger.debugf("syncAllUsers for federation provider %s was ignored as it's already in progress", fedProvider.getDisplayName());
+            return UserFederationSyncResult.ignored();
+        } else {
+            return holder.result.getResult();
+        }
     }
 
-    public void refreshPeriodicSyncForProvider(final KeycloakSessionFactory sessionFactory, TimerProvider timer, final UserFederationProviderModel fedProvider, final String realmId) {
+    public UserFederationSyncResult syncChangedUsers(final KeycloakSessionFactory sessionFactory, final String realmId, final UserFederationProviderModel fedProvider) {
+        final Holder holder = new Holder();
+
+        // Ensure not executed concurrently on this or any other cluster node
+        KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
+
+            @Override
+            public void run(KeycloakSession session) {
+                ClusterProvider clusterProvider = session.getProvider(ClusterProvider.class);
+                // shared key for "full" and "changed" . Improve if needed
+                String taskKey = fedProvider.getId() + "::sync";
+
+                // 30 seconds minimal timeout for now
+                int timeout = Math.max(30, fedProvider.getChangedSyncPeriod());
+                holder.result = clusterProvider.executeIfNotExecuted(taskKey, timeout, new Callable<UserFederationSyncResult>() {
+
+                    @Override
+                    public UserFederationSyncResult call() throws Exception {
+                        final UserFederationProviderFactory fedProviderFactory = (UserFederationProviderFactory) sessionFactory.getProviderFactory(UserFederationProvider.class, fedProvider.getProviderName());
+
+                        // See when we did last sync.
+                        int oldLastSync = fedProvider.getLastSync();
+                        updateLastSyncInterval(sessionFactory, fedProvider, realmId);
+                        return fedProviderFactory.syncChangedUsers(sessionFactory, realmId, fedProvider, Time.toDate(oldLastSync));
+                    }
+
+                });
+            }
+
+        });
+
+        if (holder.result == null || !holder.result.isExecuted()) {
+            logger.debugf("syncChangedUsers for federation provider %s was ignored as it's already in progress", fedProvider.getDisplayName());
+            return UserFederationSyncResult.ignored();
+        } else {
+            return holder.result.getResult();
+        }
+    }
+
+
+    // Ensure all cluster nodes are notified
+    public void notifyToRefreshPeriodicSync(KeycloakSession session, RealmModel realm, UserFederationProviderModel federationProvider, boolean removed) {
+        FederationProviderClusterEvent event = FederationProviderClusterEvent.createEvent(removed, realm.getId(), federationProvider);
+        session.getProvider(ClusterProvider.class).notify(FEDERATION_TASK_KEY, event);
+    }
+
+
+    // Executed once it receives notification that some UserFederationProvider was created or updated
+    protected void refreshPeriodicSyncForProvider(final KeycloakSessionFactory sessionFactory, TimerProvider timer, final UserFederationProviderModel fedProvider, final String realmId) {
+        logger.debugf("Going to refresh periodic sync for provider '%s' . Full sync period: %d , changed users sync period: %d",
+                fedProvider.getDisplayName(), fedProvider.getFullSyncPeriod(), fedProvider.getChangedSyncPeriod());
+
         if (fedProvider.getFullSyncPeriod() > 0) {
             // We want periodic full sync for this provider
             timer.schedule(new Runnable() {
@@ -84,7 +172,12 @@ public class UsersSyncManager {
                 @Override
                 public void run() {
                     try {
-                        syncAllUsers(sessionFactory, realmId, fedProvider);
+                        boolean shouldPerformSync = shouldPerformNewPeriodicSync(fedProvider.getLastSync(), fedProvider.getChangedSyncPeriod());
+                        if (shouldPerformSync) {
+                            syncAllUsers(sessionFactory, realmId, fedProvider);
+                        } else {
+                            logger.debugf("Ignored periodic full sync with federation provider %s due small time since last sync", fedProvider.getDisplayName());
+                        }
                     } catch (Throwable t) {
                         logger.errorDuringFullUserSync(t);
                     }
@@ -102,7 +195,12 @@ public class UsersSyncManager {
                 @Override
                 public void run() {
                     try {
-                        syncChangedUsers(sessionFactory, realmId, fedProvider);
+                        boolean shouldPerformSync = shouldPerformNewPeriodicSync(fedProvider.getLastSync(), fedProvider.getChangedSyncPeriod());
+                        if (shouldPerformSync) {
+                            syncChangedUsers(sessionFactory, realmId, fedProvider);
+                        } else {
+                            logger.debugf("Ignored periodic changed-users sync with federation provider %s due small time since last sync", fedProvider.getDisplayName());
+                        }
                     } catch (Throwable t) {
                         logger.errorDuringChangedUserSync(t);
                     }
@@ -115,7 +213,21 @@ public class UsersSyncManager {
         }
     }
 
-    public void removePeriodicSyncForProvider(TimerProvider timer, final UserFederationProviderModel fedProvider) {
+    // Skip syncing if there is short time since last sync time.
+    private boolean shouldPerformNewPeriodicSync(int lastSyncTime, int period) {
+        if (lastSyncTime <= 0) {
+            return true;
+        }
+
+        int currentTime = Time.currentTime();
+        int timeSinceLastSync = currentTime - lastSyncTime;
+
+        return (timeSinceLastSync * 2 > period);
+    }
+
+    // Executed once it receives notification that some UserFederationProvider was removed
+    protected void removePeriodicSyncForProvider(TimerProvider timer, UserFederationProviderModel fedProvider) {
+        logger.debugf("Removing periodic sync for provider %s", fedProvider.getDisplayName());
         timer.cancelTask(fedProvider.getId() + "-FULL");
         timer.cancelTask(fedProvider.getId() + "-CHANGED");
     }
@@ -144,4 +256,73 @@ public class UsersSyncManager {
         });
     }
 
+
+    private class UserFederationClusterListener implements ClusterListener {
+
+        private final KeycloakSessionFactory sessionFactory;
+
+        public UserFederationClusterListener(KeycloakSessionFactory sessionFactory) {
+            this.sessionFactory = sessionFactory;
+        }
+
+        @Override
+        public void run(ClusterEvent event) {
+            final FederationProviderClusterEvent fedEvent = (FederationProviderClusterEvent) event;
+            KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
+
+                @Override
+                public void run(KeycloakSession session) {
+                    TimerProvider timer = session.getProvider(TimerProvider.class);
+                    if (fedEvent.isRemoved()) {
+                        removePeriodicSyncForProvider(timer, fedEvent.getFederationProvider());
+                    } else {
+                        refreshPeriodicSyncForProvider(sessionFactory, timer, fedEvent.getFederationProvider(), fedEvent.getRealmId());
+                    }
+                }
+
+            });
+        }
+    }
+
+
+    // Send to cluster during each update or remove of federationProvider, so all nodes can update sync periods
+    public static class FederationProviderClusterEvent implements ClusterEvent {
+
+        private boolean removed;
+        private String realmId;
+        private UserFederationProviderModel federationProvider;
+
+        public boolean isRemoved() {
+            return removed;
+        }
+
+        public void setRemoved(boolean removed) {
+            this.removed = removed;
+        }
+
+        public String getRealmId() {
+            return realmId;
+        }
+
+        public void setRealmId(String realmId) {
+            this.realmId = realmId;
+        }
+
+        public UserFederationProviderModel getFederationProvider() {
+            return federationProvider;
+        }
+
+        public void setFederationProvider(UserFederationProviderModel federationProvider) {
+            this.federationProvider = federationProvider;
+        }
+
+        public static FederationProviderClusterEvent createEvent(boolean removed, String realmId, UserFederationProviderModel fedProvider) {
+            FederationProviderClusterEvent notification = new FederationProviderClusterEvent();
+            notification.setRemoved(removed);
+            notification.setRealmId(realmId);
+            notification.setFederationProvider(fedProvider);
+            return notification;
+        }
+    }
+
 }

services/src/main/java/org/keycloak/services/resources/KeycloakApplication.java

@@ -38,6 +38,7 @@ import org.keycloak.services.managers.UsersSyncManager;
 import org.keycloak.services.resources.admin.AdminRoot;
 import org.keycloak.services.scheduled.ClearExpiredEvents;
 import org.keycloak.services.scheduled.ClearExpiredUserSessions;
+import org.keycloak.services.scheduled.ClusterAwareScheduledTaskRunner;
 import org.keycloak.services.scheduled.ScheduledTaskRunner;
 import org.keycloak.services.util.JsonConfigProvider;
 import org.keycloak.services.util.ObjectMapperResolver;
@@ -217,8 +218,8 @@ public class KeycloakApplication extends Application {
         KeycloakSession session = sessionFactory.create();
         try {
             TimerProvider timer = session.getProvider(TimerProvider.class);
-            timer.schedule(new ScheduledTaskRunner(sessionFactory, new ClearExpiredEvents()), interval, "ClearExpiredEvents");
-            timer.schedule(new ScheduledTaskRunner(sessionFactory, new ClearExpiredUserSessions()), interval, "ClearExpiredUserSessions");
+            timer.schedule(new ClusterAwareScheduledTaskRunner(sessionFactory, new ClearExpiredEvents(), interval), interval, "ClearExpiredEvents");
+            timer.schedule(new ClusterAwareScheduledTaskRunner(sessionFactory, new ClearExpiredUserSessions(), interval), interval, "ClearExpiredUserSessions");
             new UsersSyncManager().bootstrapPeriodic(sessionFactory, timer);
         } finally {
             session.close();

services/src/main/java/org/keycloak/services/resources/admin/RealmAdminResource.java

@@ -242,7 +242,7 @@ public class RealmAdminResource {
             List<UserFederationProviderModel> federationProviders = realm.getUserFederationProviders();
             UsersSyncManager usersSyncManager = new UsersSyncManager();
             for (final UserFederationProviderModel fedProvider : federationProviders) {
-                usersSyncManager.refreshPeriodicSyncForProvider(session.getKeycloakSessionFactory(), session.getProvider(TimerProvider.class), fedProvider, realm.getId());
+                usersSyncManager.notifyToRefreshPeriodicSync(session, realm, fedProvider, false);
             }
 
             adminEvent.operation(OperationType.UPDATE).representation(rep).success();

services/src/main/java/org/keycloak/services/resources/admin/UserFederationProviderResource.java

@@ -106,7 +106,7 @@ public class UserFederationProviderResource {
         UserFederationProviderModel model = new UserFederationProviderModel(rep.getId(), rep.getProviderName(), rep.getConfig(), rep.getPriority(), displayName,
                 rep.getFullSyncPeriod(), rep.getChangedSyncPeriod(), rep.getLastSync());
         realm.updateUserFederationProvider(model);
-        new UsersSyncManager().refreshPeriodicSyncForProvider(session.getKeycloakSessionFactory(), session.getProvider(TimerProvider.class), model, realm.getId());
+        new UsersSyncManager().notifyToRefreshPeriodicSync(session, realm, model, false);
         boolean kerberosCredsAdded = UserFederationProvidersResource.checkKerberosCredential(session, realm, model);
         if (kerberosCredsAdded) {
             logger.addedKerberosToRealmCredentials();
@@ -138,7 +138,7 @@ public class UserFederationProviderResource {
         auth.requireManage();
 
         realm.removeUserFederationProvider(this.federationProviderModel);
-        new UsersSyncManager().removePeriodicSyncForProvider(session.getProvider(TimerProvider.class), this.federationProviderModel);
+        new UsersSyncManager().notifyToRefreshPeriodicSync(session, realm, this.federationProviderModel, true);
 
         adminEvent.operation(OperationType.DELETE).resourcePath(uriInfo).success();
 

services/src/main/java/org/keycloak/services/resources/admin/UserFederationProvidersResource.java

@@ -178,7 +178,7 @@ public class UserFederationProvidersResource {
         }
         UserFederationProviderModel model = realm.addUserFederationProvider(rep.getProviderName(), rep.getConfig(), rep.getPriority(), displayName,
                 rep.getFullSyncPeriod(), rep.getChangedSyncPeriod(), rep.getLastSync());
-        new UsersSyncManager().refreshPeriodicSyncForProvider(session.getKeycloakSessionFactory(), session.getProvider(TimerProvider.class), model, realm.getId());
+        new UsersSyncManager().notifyToRefreshPeriodicSync(session, realm, model, false);
         boolean kerberosCredsAdded = checkKerberosCredential(session, realm, model);
         if (kerberosCredsAdded) {
             logger.addedKerberosToRealmCredentials();

services/src/main/java/org/keycloak/services/scheduled/ClusterAwareScheduledTaskRunner.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.services.scheduled;
+
+import java.util.concurrent.Callable;
+
+import org.keycloak.cluster.ClusterProvider;
+import org.keycloak.cluster.ExecutionResult;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.KeycloakSessionFactory;
+
+/**
+ * Ensures that there are not concurrent executions of same task (either on this host or any other cluster host)
+ *
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class ClusterAwareScheduledTaskRunner extends ScheduledTaskRunner {
+
+    private final int intervalSecs;
+
+    public ClusterAwareScheduledTaskRunner(KeycloakSessionFactory sessionFactory, ScheduledTask task, long intervalMillis) {
+        super(sessionFactory, task);
+        this.intervalSecs = (int) (intervalMillis / 1000);
+    }
+
+    @Override
+    protected void runTask(final KeycloakSession session) {
+        session.getTransaction().begin();
+
+        ClusterProvider clusterProvider = session.getProvider(ClusterProvider.class);
+        String taskKey = task.getClass().getSimpleName();
+
+        ExecutionResult<Void> result = clusterProvider.executeIfNotExecuted(taskKey, intervalSecs, new Callable<Void>() {
+
+            @Override
+            public Void call() throws Exception {
+                task.run(session);
+                return null;
+            }
+
+        });
+
+        session.getTransaction().commit();
+
+        if (result.isExecuted()) {
+            logger.debugf("Executed scheduled task %s", taskKey);
+        } else {
+            logger.debugf("Skipped execution of task %s as other cluster node is executing it", taskKey);
+        }
+    }
+
+
+}

services/src/main/java/org/keycloak/services/scheduled/ScheduledTaskRunner.java

@@ -17,6 +17,10 @@
 
 package org.keycloak.services.scheduled;
 
+import java.util.concurrent.Callable;
+
+import org.keycloak.cluster.ClusterProvider;
+import org.keycloak.cluster.ExecutionResult;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
 import org.keycloak.services.ServicesLogger;
@@ -26,10 +30,10 @@ import org.keycloak.services.ServicesLogger;
  */
 public class ScheduledTaskRunner implements Runnable {
 
-    private static final ServicesLogger logger = ServicesLogger.ROOT_LOGGER;
+    protected static final ServicesLogger logger = ServicesLogger.ROOT_LOGGER;
 
-    private final KeycloakSessionFactory sessionFactory;
-    private final ScheduledTask task;
+    protected final KeycloakSessionFactory sessionFactory;
+    protected final ScheduledTask task;
 
     public ScheduledTaskRunner(KeycloakSessionFactory sessionFactory, ScheduledTask task) {
         this.sessionFactory = sessionFactory;
@@ -40,11 +44,7 @@ public class ScheduledTaskRunner implements Runnable {
     public void run() {
         KeycloakSession session = sessionFactory.create();
         try {
-            session.getTransaction().begin();
-            task.run(session);
-            session.getTransaction().commit();
-
-            logger.debug("Executed scheduled task " + task.getClass().getSimpleName());
+            runTask(session);
         } catch (Throwable t) {
             logger.failedToRunScheduledTask(t, task.getClass().getSimpleName());
 
@@ -58,4 +58,12 @@ public class ScheduledTaskRunner implements Runnable {
         }
     }
 
+    protected void runTask(KeycloakSession session) {
+        session.getTransaction().begin();
+        task.run(session);
+        session.getTransaction().commit();
+
+        logger.debug("Executed scheduled task " + task.getClass().getSimpleName());
+    }
+
 }

services/src/main/java/org/keycloak/theme/ExtendingThemeManager.java

@@ -19,6 +19,7 @@ package org.keycloak.theme;
 
 import org.jboss.logging.Logger;
 import org.keycloak.Config;
+import org.keycloak.common.Version;
 import org.keycloak.models.KeycloakSession;
 
 import java.io.IOException;
@@ -42,7 +43,7 @@ public class ExtendingThemeManager implements ThemeProvider {
     public ExtendingThemeManager(KeycloakSession session, ConcurrentHashMap<ExtendingThemeManagerFactory.ThemeKey, Theme> themeCache) {
         this.session = session;
         this.themeCache = themeCache;
-        this.defaultTheme = Config.scope("theme").get("default", "keycloak");
+        this.defaultTheme = Config.scope("theme").get("default", Version.NAME.toLowerCase());
     }
 
     private List<ThemeProvider> getProviders() {

testsuite/integration-arquillian/servers/wildfly/pom.xml

@@ -415,6 +415,11 @@
         
         <profile>
             <id>auth-server-wildfly-cluster</id>
+            <properties>
+                <session.cache.owners>1</session.cache.owners>
+                <offline.session.cache.owners>1</offline.session.cache.owners>
+                <login.failure.cache.owners>1</login.failure.cache.owners>
+            </properties>
             <build>
                 <plugins>
                     <plugin>
@@ -448,6 +453,28 @@
                                                 </parameter>
                                             </parameters>
                                         </transformationSet>
+                                        <transformationSet>
+                                            <dir>${keycloak.server.home}/standalone/configuration</dir>
+                                            <includes>
+                                                <include>standalone-ha.xml</include>
+                                            </includes>
+                                            <stylesheet>src/main/xslt/ispn-cache-owners.xsl</stylesheet>
+                                            <outputDir>${keycloak.server.home}/standalone/configuration</outputDir>
+                                            <parameters>
+                                                <parameter>
+                                                    <name>sessionCacheOwners</name>
+                                                    <value>${session.cache.owners}</value>
+                                                </parameter>
+                                                <parameter>
+                                                    <name>offlineSessionCacheOwners</name>
+                                                    <value>${offline.session.cache.owners}</value>
+                                                </parameter>
+                                                <parameter>
+                                                    <name>loginFailureCacheOwners</name>
+                                                    <value>${login.failure.cache.owners}</value>
+                                                </parameter>
+                                            </parameters>
+                                        </transformationSet>
                                     </transformationSets>
                                 </configuration>
                             </execution>

testsuite/integration-arquillian/servers/wildfly/src/main/xslt/ispn-cache-owners.xsl

@@ -0,0 +1,40 @@
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+                xmlns:xalan="http://xml.apache.org/xalan"
+                xmlns:j="urn:jboss:domain:4.0"
+                xmlns:i="urn:jboss:domain:infinispan:4.0"
+                version="2.0"
+                exclude-result-prefixes="xalan i">
+
+    <xsl:output method="xml" version="1.0" encoding="UTF-8" indent="yes" xalan:indent-amount="4" standalone="no"/>
+    <xsl:strip-space elements="*"/>
+
+    <xsl:variable name="nsDS" select="'urn:jboss:domain:datasources:'"/>
+    
+    <xsl:param name="sessionCacheOwners" select="'1'"/>
+    <xsl:param name="offlineSessionCacheOwners" select="'1'"/>
+    <xsl:param name="loginFailureCacheOwners" select="'1'"/>
+
+    <xsl:template match="//i:cache-container/i:distributed-cache[@name='sessions']/@owners">
+        <xsl:attribute name="owners">
+            <xsl:value-of select="$sessionCacheOwners"/>
+        </xsl:attribute>
+    </xsl:template>
+    <xsl:template match="//i:cache-container/i:distributed-cache[@name='offlineSessions']/@owners">
+        <xsl:attribute name="owners">
+            <xsl:value-of select="$offlineSessionCacheOwners"/>
+        </xsl:attribute>
+    </xsl:template>
+    <xsl:template match="//i:cache-container/i:distributed-cache[@name='loginFailures']/@owners">
+        <xsl:attribute name="owners">
+            <xsl:value-of select="$loginFailureCacheOwners"/>
+        </xsl:attribute>
+    </xsl:template>
+
+    <!-- Copy everything else. -->
+    <xsl:template match="@*|node()">
+        <xsl:copy>
+            <xsl:apply-templates select="@*|node()" />
+        </xsl:copy>
+    </xsl:template>
+
+</xsl:stylesheet>

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/adapter/page/SAMLPostSigExample.java

@@ -29,7 +29,7 @@ import java.net.URL;
  * @author mhajas
  */
 public class SAMLPostSigExample extends AbstractPageWithInjectedUrl {
-    public static final String DEPLOYMENT_NAME = "saml-post-signatures";
+    public static final String DEPLOYMENT_NAME = "sales-post-sig";
 
     @ArquillianResource
     @OperateOnDeployment(DEPLOYMENT_NAME)

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/admin/ApiUtil.java

@@ -30,8 +30,10 @@ import java.net.URI;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import org.apache.commons.lang.builder.EqualsBuilder;
 
 import static org.keycloak.representations.idm.CredentialRepresentation.PASSWORD;
+import org.keycloak.representations.idm.GroupRepresentation;
 
 /**
  * Created by st on 28.05.15.
@@ -39,7 +41,7 @@ import static org.keycloak.representations.idm.CredentialRepresentation.PASSWORD
 public class ApiUtil {
     
     private static final Logger log = Logger.getLogger(ApiUtil.class);
-
+    
     public static String getCreatedId(Response response) {
         URI location = response.getLocation();
         if (location == null) {
@@ -48,7 +50,7 @@ public class ApiUtil {
         String path = location.getPath();
         return path.substring(path.lastIndexOf('/') + 1);
     }
-
+    
     public static ClientResource findClientResourceByClientId(RealmResource realm, String clientId) {
         for (ClientRepresentation c : realm.clients().findAll()) {
             if (c.getClientId().equals(clientId)) {
@@ -57,7 +59,7 @@ public class ApiUtil {
         }
         return null;
     }
-
+    
     public static ClientResource findClientResourceByName(RealmResource realm, String name) {
         for (ClientRepresentation c : realm.clients().findAll()) {
             if (c.getName().equals(name)) {
@@ -66,7 +68,7 @@ public class ApiUtil {
         }
         return null;
     }
-
+    
     public static ClientRepresentation findClientByClientId(RealmResource realm, String clientId) {
         ClientRepresentation client = null;
         for (ClientRepresentation c : realm.clients().findAll()) {
@@ -76,7 +78,7 @@ public class ApiUtil {
         }
         return client;
     }
-
+    
     public static UserRepresentation findUserByUsername(RealmResource realm, String username) {
         UserRepresentation user = null;
         List<UserRepresentation> ur = realm.users().search(username, null, null);
@@ -85,7 +87,7 @@ public class ApiUtil {
         }
         return user;
     }
-
+    
     public static String createUserWithAdminClient(RealmResource realm, UserRepresentation user) {
         Response response = realm.users().create(user);
         String createdId = getCreatedId(response);
@@ -98,7 +100,7 @@ public class ApiUtil {
         resetUserPassword(realm.users().get(id), password, false);
         return id;
     }
-
+    
     public static void resetUserPassword(UserResource userResource, String newPassword, boolean temporary) {
         CredentialRepresentation newCredential = new CredentialRepresentation();
         newCredential.setType(PASSWORD);
@@ -106,7 +108,7 @@ public class ApiUtil {
         newCredential.setTemporary(temporary);
         userResource.resetPassword(newCredential);
     }
-
+    
     public static void assignClientRoles(RealmResource realm, String userId, String clientName, String... roles) {
         String realmName = realm.toRepresentation().getRealm();
         String clientId = "";
@@ -118,21 +120,32 @@ public class ApiUtil {
         
         if (!clientId.isEmpty()) {
             ClientResource clientResource = realm.clients().get(clientId);
-
+            
             List<RoleRepresentation> roleRepresentations = new ArrayList<>();
             for (String roleName : roles) {
                 RoleRepresentation role = clientResource.roles().get(roleName).toRepresentation();
                 roleRepresentations.add(role);
             }
-
+            
             UserResource userResource = realm.users().get(userId);
-            log.debug("assigning roles: " + Arrays.toString(roles) + " to user: \"" + 
-                    userResource.toRepresentation().getUsername() + "\" of client: \"" + 
-                    clientName + "\" in realm: \"" + realmName + "\"");
+            log.debug("assigning roles: " + Arrays.toString(roles) + " to user: \""
+                    + userResource.toRepresentation().getUsername() + "\" of client: \""
+                    + clientName + "\" in realm: \"" + realmName + "\"");
             userResource.roles().clientLevel(clientId).add(roleRepresentations);
         } else {
             log.warn("client with name " + clientName + "doesn't exist in realm " + realmName);
         }
     }
-
+    
+    public static boolean groupContainsSubgroup(GroupRepresentation group, GroupRepresentation subgroup) {
+        boolean contains = false;
+        for (GroupRepresentation sg : group.getSubGroups()) {
+            if (subgroup.getId().equals(sg.getId())) {
+                contains = true;
+                break;
+            }
+        }
+        return contains;
+    }
+    
 }

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/AppServerTestEnricher.java

@@ -1,9 +1,5 @@
 package org.keycloak.testsuite.arquillian;
 
-import java.io.File;
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URL;
 import org.jboss.arquillian.container.spi.event.container.BeforeDeploy;
 import org.jboss.arquillian.container.test.api.ContainerController;
 import org.jboss.arquillian.core.api.Instance;
@@ -13,12 +9,18 @@ import org.jboss.arquillian.core.api.annotation.Observes;
 import org.jboss.arquillian.test.spi.annotation.ClassScoped;
 import org.jboss.arquillian.test.spi.event.suite.BeforeClass;
 import org.jboss.logging.Logger;
-import static org.keycloak.testsuite.arquillian.AuthServerTestEnricher.getAuthServerContextRoot;
-import static org.keycloak.testsuite.arquillian.AuthServerTestEnricher.getAuthServerQualifier;
 import org.keycloak.testsuite.arquillian.annotation.AdapterLibsLocationProperty;
 import org.keycloak.testsuite.arquillian.annotation.AppServerContainer;
-import static org.keycloak.testsuite.util.IOUtil.execCommand;
 import org.keycloak.testsuite.util.LogChecker;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+
+import static org.keycloak.testsuite.arquillian.AuthServerTestEnricher.getAuthServerContextRoot;
+import static org.keycloak.testsuite.arquillian.AuthServerTestEnricher.getAuthServerQualifier;
+import static org.keycloak.testsuite.util.IOUtil.execCommand;
 import static org.keycloak.testsuite.util.WaitUtils.pause;
 
 /**
@@ -158,7 +160,9 @@ public class AppServerTestEnricher {
                 execCommand(command + " --connect --command=reload" + controllerArg, bin);
                 log.info("Container restarted");
                 pause(5000);
-                LogChecker.checkJBossServerLog(jbossHomePath);
+                if (System.getProperty("app.server.log.check","true").equals("true")) {
+                    LogChecker.checkJBossServerLog(jbossHomePath);
+                }
             }
 
             appServerInfo.setAdapterLibsInstalled(true);

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/auth/page/login/Login.java

@@ -35,11 +35,12 @@ public abstract class Login extends AuthRealm {
     public static final String PROTOCOL = "protocol";
     public static final String OIDC = "openid-connect";
     public static final String SAML = "saml";
+    public static final String LOGIN_ACTION = "login-action";
 
     @Override
     public UriBuilder createUriBuilder() {
         return super.createUriBuilder()
-                .path("protocol/{" + PROTOCOL + "}" + (getProtocol().equals(OIDC) ? "/auth" : ""));
+                .path((getProtocol().equals(OIDC) || getProtocol().equals(SAML)) ? "protocol/" : "" + "{" + PROTOCOL + "}" + (getProtocol().equals(OIDC) ? "/auth" : ""));
     }
     
     public void setProtocol(String protocol) {

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/auth/page/login/SAMLPostLogin.java

@@ -0,0 +1,10 @@
+package org.keycloak.testsuite.auth.page.login;
+
+/**
+ * @author mhajas
+ */
+public class SAMLPostLogin extends Login {
+    SAMLPostLogin() {
+        setProtocol(LOGIN_ACTION);
+    }
+}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/auth/page/login/SAMLRedirectLogin.java

@@ -20,8 +20,8 @@ package org.keycloak.testsuite.auth.page.login;
 /**
  * @author mhajas
  */
-public class SAMLLogin extends Login {
-    SAMLLogin() {
+public class SAMLRedirectLogin extends Login {
+    SAMLRedirectLogin() {
         setProtocol(SAML);
     }
 }

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/AbstractAuthTest.java

@@ -23,7 +23,8 @@ import org.keycloak.representations.idm.RealmRepresentation;
 import org.keycloak.representations.idm.UserRepresentation;
 import org.keycloak.testsuite.auth.page.AuthRealm;
 import org.keycloak.testsuite.auth.page.login.OIDCLogin;
-import org.keycloak.testsuite.auth.page.login.SAMLLogin;
+import org.keycloak.testsuite.auth.page.login.SAMLPostLogin;
+import org.keycloak.testsuite.auth.page.login.SAMLRedirectLogin;
 import org.openqa.selenium.Cookie;
 
 import java.text.MessageFormat;
@@ -47,7 +48,10 @@ public abstract class AbstractAuthTest extends AbstractKeycloakTest {
     protected OIDCLogin testRealmLoginPage;
 
     @Page
-    protected SAMLLogin testRealmSAMLLoginPage;
+    protected SAMLPostLogin testRealmSAMLPostLoginPage;
+
+    @Page
+    protected SAMLRedirectLogin testRealmSAMLRedirectLoginPage;
 
     protected UserRepresentation testUser;
 

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/adapter/AbstractAdapterTest.java

@@ -135,7 +135,7 @@ public abstract class AbstractAdapterTest extends AbstractAuthTest {
                         masterUrl = client.getBaseUrl();
                     }
                     masterUrl = masterUrl.replaceFirst(regex, replacement);
-                    client.setAdminUrl(masterUrl);
+                    client.setAdminUrl(masterUrl + ((!masterUrl.endsWith("/saml")) ? "/saml" : ""));
                 }
             }
         }

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/adapter/example/AbstractSAMLExampleAdapterTest.java

@@ -60,7 +60,8 @@ public abstract class AbstractSAMLExampleAdapterTest extends AbstractExampleAdap
     public void setDefaultPageUriParameters() {
         super.setDefaultPageUriParameters();
         testRealmPage.setAuthRealm(SAMLDEMO);
-        testRealmSAMLLoginPage.setAuthRealm(SAMLDEMO);
+        testRealmSAMLRedirectLoginPage.setAuthRealm(SAMLDEMO);
+        testRealmSAMLPostLoginPage.setAuthRealm(SAMLDEMO);
     }
 
     @Deployment(name = SAMLPostSigExample.DEPLOYMENT_NAME)
@@ -81,41 +82,41 @@ public abstract class AbstractSAMLExampleAdapterTest extends AbstractExampleAdap
     @Test
     public void samlPostWithSignatureExampleTest() {
         samlPostSigExamplePage.navigateTo();
-        testRealmSAMLLoginPage.form().login(bburkeUser);
+        testRealmSAMLPostLoginPage.form().login(bburkeUser);
 
         assertTrue(driver.getPageSource().contains("Welcome to the Sales Tool, " + bburkeUser.getUsername()));
 
         samlPostSigExamplePage.logout();
 
         samlPostSigExamplePage.navigateTo();
-        URLAssert.assertCurrentUrlStartsWith(testRealmSAMLLoginPage);
+        URLAssert.assertCurrentUrlStartsWith(testRealmSAMLPostLoginPage);
     }
 
     @Test
     public void samlPostWithEncryptionExampleTest() {
         samlPostEncExamplePage.navigateTo();
 
-        testRealmSAMLLoginPage.form().login(bburkeUser);
+        testRealmSAMLPostLoginPage.form().login(bburkeUser);
 
         assertTrue(driver.getPageSource().contains("Welcome to the Sales Tool, " + bburkeUser.getUsername()));
 
         samlPostEncExamplePage.logout();
 
         samlPostEncExamplePage.navigateTo();
-        URLAssert.assertCurrentUrlStartsWith(testRealmSAMLLoginPage);
+        URLAssert.assertCurrentUrlStartsWith(testRealmSAMLPostLoginPage);
     }
 
     @Test
     public void samlRedirectWithSignatureExampleTest() {
         samlRedirectSigExamplePage.navigateTo();
 
-        testRealmSAMLLoginPage.form().login(bburkeUser);
+        testRealmSAMLRedirectLoginPage.form().login(bburkeUser);
 
         assertTrue(driver.getPageSource().contains("Welcome to the Employee Tool,"));
 
         samlRedirectSigExamplePage.logout();
 
         samlRedirectSigExamplePage.navigateTo();
-        URLAssert.assertCurrentUrlStartsWith(testRealmSAMLLoginPage);
+        URLAssert.assertCurrentUrlStartsWith(testRealmSAMLRedirectLoginPage);
     }
 }

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/adapter/servlet/AbstractSAMLServletsAdapterTest.java

@@ -33,9 +33,7 @@ import org.w3c.dom.Document;
 import javax.ws.rs.core.Response;
 import java.util.List;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
 import static org.keycloak.testsuite.auth.page.AuthRealm.SAMLSERVLETDEMO;
 import static org.keycloak.testsuite.util.IOUtil.*;
 import static org.keycloak.testsuite.util.URLAssert.assertCurrentUrlStartsWith;
@@ -157,7 +155,8 @@ public abstract class AbstractSAMLServletsAdapterTest extends AbstractServletsAd
     public void setDefaultPageUriParameters() {
         super.setDefaultPageUriParameters();
         testRealmPage.setAuthRealm(SAMLSERVLETDEMO);
-        testRealmSAMLLoginPage.setAuthRealm(SAMLSERVLETDEMO);
+        testRealmSAMLRedirectLoginPage.setAuthRealm(SAMLSERVLETDEMO);
+        testRealmSAMLPostLoginPage.setAuthRealm(SAMLSERVLETDEMO);
     }
 
     @Test
@@ -177,7 +176,7 @@ public abstract class AbstractSAMLServletsAdapterTest extends AbstractServletsAd
     @Test
     public void unauthorizedSSOTest() {
         salesPostServletPage.navigateTo();
-        testRealmSAMLLoginPage.form().login("unauthorized", "password");
+        testRealmSAMLRedirectLoginPage.form().login("unauthorized", "password");
 
         assertFalse(driver.getPageSource().contains("principal="));
         assertTrue(driver.getPageSource().contains("Forbidden") || driver.getPageSource().contains("Status 403"));
@@ -200,7 +199,7 @@ public abstract class AbstractSAMLServletsAdapterTest extends AbstractServletsAd
     @Test
     public void singleLoginAndLogoutSAMLTest() {
         salesPostServletPage.navigateTo();
-        testRealmSAMLLoginPage.form().login(bburkeUser);
+        testRealmSAMLRedirectLoginPage.form().login(bburkeUser);
         assertTrue(driver.getPageSource().contains("principal=bburke"));
 
         salesPostSigServletPage.navigateTo();
@@ -215,16 +214,16 @@ public abstract class AbstractSAMLServletsAdapterTest extends AbstractServletsAd
         employeeSigFrontServletPage.logout();
 
         employeeSigFrontServletPage.navigateTo();
-        assertCurrentUrlStartsWith(testRealmSAMLLoginPage);
+        assertCurrentUrlStartsWith(testRealmSAMLRedirectLoginPage);
 
         employeeSigServletPage.navigateTo();
-        assertCurrentUrlStartsWith(testRealmSAMLLoginPage);
+        assertCurrentUrlStartsWith(testRealmSAMLRedirectLoginPage);
 
         salesPostPassiveServletPage.navigateTo();
-        assertTrue(driver.getPageSource().contains("Forbidden") || driver.getPageSource().contains("<body><pre></pre></body>"));
+        assertTrue(driver.getPageSource().contains("Forbidden") || driver.getPageSource().contains("<body></body>"));
 
         salesPostSigEmailServletPage.navigateTo();
-        assertCurrentUrlStartsWith(testRealmSAMLLoginPage);
+        assertCurrentUrlStartsWith(testRealmSAMLPostLoginPage);
     }
 
     @Test
@@ -236,7 +235,7 @@ public abstract class AbstractSAMLServletsAdapterTest extends AbstractServletsAd
     @Test
     public void badRealmSalesPostSigTest() {
         badRealmSalesPostSigServletPage.navigateTo();
-        testRealmSAMLLoginPage.form().login(bburkeUser);
+        testRealmSAMLRedirectLoginPage.form().login(bburkeUser);
 
         //Different 403 status page on EAP and Wildfly
         assertTrue(driver.getPageSource().contains("Forbidden") || driver.getPageSource().contains("Status 403"));
@@ -245,14 +244,14 @@ public abstract class AbstractSAMLServletsAdapterTest extends AbstractServletsAd
     @Test
     public void employee2Test() {
         employee2ServletPage.navigateTo();
-        testRealmSAMLLoginPage.form().login(bburkeUser);
+        testRealmSAMLPostLoginPage.form().login(bburkeUser);
         assertTrue(driver.getPageSource().contains("principal=bburke"));
 
         employee2ServletPage.logout();
         employee2ServletPage.navigateTo();
-        assertCurrentUrlStartsWith(testRealmSAMLLoginPage);
+        assertCurrentUrlStartsWith(testRealmSAMLPostLoginPage);
 
-        testRealmSAMLLoginPage.form().login("unauthorized", "password");
+        testRealmSAMLPostLoginPage.form().login("unauthorized", "password");
         assertFalse(driver.getPageSource().contains("principal="));
         //Different 403 status page on EAP and Wildfly
         assertTrue(driver.getPageSource().contains("Forbidden") || driver.getPageSource().contains("Status 403"));
@@ -262,14 +261,14 @@ public abstract class AbstractSAMLServletsAdapterTest extends AbstractServletsAd
     @Test
     public void employeeSigTest() {
         employeeSigServletPage.navigateTo();
-        testRealmSAMLLoginPage.form().login(bburkeUser);
+        testRealmSAMLRedirectLoginPage.form().login(bburkeUser);
         assertTrue(driver.getPageSource().contains("principal=bburke"));
 
         employeeSigServletPage.logout();
         employeeSigServletPage.navigateTo();
-        assertCurrentUrlStartsWith(testRealmSAMLLoginPage);
+        assertCurrentUrlStartsWith(testRealmSAMLRedirectLoginPage);
 
-        testRealmSAMLLoginPage.form().login("unauthorized", "password");
+        testRealmSAMLRedirectLoginPage.form().login("unauthorized", "password");
         assertFalse(driver.getPageSource().contains("principal="));
         //Different 403 status page on EAP and Wildfly
         assertTrue(driver.getPageSource().contains("Forbidden") || driver.getPageSource().contains("Status 403"));
@@ -279,14 +278,14 @@ public abstract class AbstractSAMLServletsAdapterTest extends AbstractServletsAd
     @Test
     public void employeeSigFrontTest() {
         employeeSigFrontServletPage.navigateTo();
-        testRealmSAMLLoginPage.form().login(bburkeUser);
+        testRealmSAMLRedirectLoginPage.form().login(bburkeUser);
         assertTrue(driver.getPageSource().contains("principal=bburke"));
 
         employeeSigFrontServletPage.logout();
         employeeSigFrontServletPage.navigateTo();
-        assertCurrentUrlStartsWith(testRealmSAMLLoginPage);
+        assertCurrentUrlStartsWith(testRealmSAMLRedirectLoginPage);
 
-        testRealmSAMLLoginPage.form().login("unauthorized", "password");
+        testRealmSAMLRedirectLoginPage.form().login("unauthorized", "password");
         assertFalse(driver.getPageSource().contains("principal="));
         //Different 403 status page on EAP and Wildfly
         assertTrue(driver.getPageSource().contains("Forbidden") || driver.getPageSource().contains("Status 403"));
@@ -301,19 +300,29 @@ public abstract class AbstractSAMLServletsAdapterTest extends AbstractServletsAd
         modifyDocElementAttribute(doc, "AssertionConsumerService", "Location", "8080", System.getProperty("app.server.http.port", null));
 
         ClientRepresentation clientRep = testRealmResource().convertClientDescription(IOUtil.documentToString(doc));
+
+        String appServerUrl;
+        if (Boolean.parseBoolean(System.getProperty("app.server.ssl.required"))) {
+            appServerUrl = "https://localhost:" + System.getProperty("app.server.https.port", "8543") + "/";
+        } else {
+            appServerUrl = "http://localhost:" + System.getProperty("app.server.http.port", "8280") + "/";
+        }
+
+        clientRep.setAdminUrl(appServerUrl + "sales-metadata/saml");
+
         Response response = testRealmResource().clients().create(clientRep);
         assertEquals(201, response.getStatus());
         response.close();
 
         salesMetadataServletPage.navigateTo();
-        testRealmSAMLLoginPage.form().login(bburkeUser);
+        testRealmSAMLPostLoginPage.form().login(bburkeUser);
         assertTrue(driver.getPageSource().contains("principal=bburke"));
 
         salesMetadataServletPage.logout();
         salesMetadataServletPage.navigateTo();
-        assertCurrentUrlStartsWith(testRealmSAMLLoginPage);
+        assertCurrentUrlStartsWith(testRealmSAMLPostLoginPage);
 
-        testRealmSAMLLoginPage.form().login("unauthorized", "password");
+        testRealmSAMLPostLoginPage.form().login("unauthorized", "password");
         assertFalse(driver.getPageSource().contains("principal="));
         //Different 403 status page on EAP and Wildfly
         assertTrue(driver.getPageSource().contains("Forbidden") || driver.getPageSource().contains("Status 403"));
@@ -323,14 +332,14 @@ public abstract class AbstractSAMLServletsAdapterTest extends AbstractServletsAd
     @Test
     public void salesPostTest() {
         salesPostServletPage.navigateTo();
-        testRealmSAMLLoginPage.form().login(bburkeUser);
+        testRealmSAMLPostLoginPage.form().login(bburkeUser);
         assertTrue(driver.getPageSource().contains("principal=bburke"));
 
         salesPostServletPage.logout();
         salesPostServletPage.navigateTo();
-        assertCurrentUrlStartsWith(testRealmSAMLLoginPage);
+        assertCurrentUrlStartsWith(testRealmSAMLPostLoginPage);
 
-        testRealmSAMLLoginPage.form().login("unauthorized", "password");
+        testRealmSAMLPostLoginPage.form().login("unauthorized", "password");
         assertFalse(driver.getPageSource().contains("principal="));
         //Different 403 status page on EAP and Wildfly
         assertTrue(driver.getPageSource().contains("Forbidden") || driver.getPageSource().contains("Status 403"));
@@ -340,14 +349,14 @@ public abstract class AbstractSAMLServletsAdapterTest extends AbstractServletsAd
     @Test
     public void salesPostEncTest() {
         salesPostEncServletPage.navigateTo();
-        testRealmSAMLLoginPage.form().login(bburkeUser);
+        testRealmSAMLPostLoginPage.form().login(bburkeUser);
         assertTrue(driver.getPageSource().contains("principal=bburke"));
 
         salesPostEncServletPage.logout();
         salesPostEncServletPage.navigateTo();
-        assertCurrentUrlStartsWith(testRealmSAMLLoginPage);
+        assertCurrentUrlStartsWith(testRealmSAMLPostLoginPage);
 
-        testRealmSAMLLoginPage.form().login("unauthorized", "password");
+        testRealmSAMLPostLoginPage.form().login("unauthorized", "password");
         assertFalse(driver.getPageSource().contains("principal="));
         //Different 403 status page on EAP and Wildfly
         assertTrue(driver.getPageSource().contains("Forbidden") || driver.getPageSource().contains("Status 403"));
@@ -358,10 +367,10 @@ public abstract class AbstractSAMLServletsAdapterTest extends AbstractServletsAd
     public void salesPostPassiveTest() {
         salesPostPassiveServletPage.navigateTo();
         //Different 403 status page on EAP and Wildfly
-        assertTrue(driver.getPageSource().contains("Forbidden") || driver.getPageSource().contains("<body><pre></pre></body>"));
+        assertTrue(driver.getPageSource().contains("Forbidden") || driver.getPageSource().contains("<body></body>"));
 
         salesPostServletPage.navigateTo();
-        testRealmSAMLLoginPage.form().login(bburkeUser);
+        testRealmSAMLRedirectLoginPage.form().login(bburkeUser);
 
         salesPostPassiveServletPage.navigateTo();
         assertTrue(driver.getPageSource().contains("principal=bburke"));
@@ -369,10 +378,10 @@ public abstract class AbstractSAMLServletsAdapterTest extends AbstractServletsAd
         salesPostPassiveServletPage.logout();
         salesPostPassiveServletPage.navigateTo();
         //Different 403 status page on EAP and Wildfly
-        assertTrue(driver.getPageSource().contains("Forbidden") || driver.getPageSource().contains("<body><pre></pre></body>"));
+        assertTrue(driver.getPageSource().contains("Forbidden") || driver.getPageSource().contains("<body></body>"));
 
         salesPostServletPage.navigateTo();
-        testRealmSAMLLoginPage.form().login("unauthorized", "password");
+        testRealmSAMLRedirectLoginPage.form().login("unauthorized", "password");
         salesPostPassiveServletPage.navigateTo();
         assertFalse(driver.getPageSource().contains("principal="));
         //Different 403 status page on EAP and Wildfly
@@ -383,14 +392,14 @@ public abstract class AbstractSAMLServletsAdapterTest extends AbstractServletsAd
     @Test
     public void salesPostSigTest() {
         salesPostEncServletPage.navigateTo();
-        testRealmSAMLLoginPage.form().login(bburkeUser);
+        testRealmSAMLPostLoginPage.form().login(bburkeUser);
         assertTrue(driver.getPageSource().contains("principal=bburke"));
 
         salesPostEncServletPage.logout();
         salesPostEncServletPage.navigateTo();
-        assertCurrentUrlStartsWith(testRealmSAMLLoginPage);
+        assertCurrentUrlStartsWith(testRealmSAMLPostLoginPage);
 
-        testRealmSAMLLoginPage.form().login("unauthorized", "password");
+        testRealmSAMLPostLoginPage.form().login("unauthorized", "password");
         assertFalse(driver.getPageSource().contains("principal="));
         //Different 403 status page on EAP and Wildfly
         assertTrue(driver.getPageSource().contains("Forbidden") || driver.getPageSource().contains("Status 403"));
@@ -400,14 +409,14 @@ public abstract class AbstractSAMLServletsAdapterTest extends AbstractServletsAd
     @Test
     public void salesPostSigEmailTest() {
         salesPostSigEmailServletPage.navigateTo();
-        testRealmSAMLLoginPage.form().login(bburkeUser);
+        testRealmSAMLPostLoginPage.form().login(bburkeUser);
         assertTrue(driver.getPageSource().contains("principal=bburke"));
 
         salesPostSigEmailServletPage.logout();
         salesPostSigEmailServletPage.navigateTo();
-        assertCurrentUrlStartsWith(testRealmSAMLLoginPage);
+        assertCurrentUrlStartsWith(testRealmSAMLPostLoginPage);
 
-        testRealmSAMLLoginPage.form().login("unauthorized", "password");
+        testRealmSAMLPostLoginPage.form().login("unauthorized", "password");
         assertFalse(driver.getPageSource().contains("principal="));
         //Different 403 status page on EAP and Wildfly
         assertTrue(driver.getPageSource().contains("Forbidden") || driver.getPageSource().contains("Status 403"));
@@ -417,15 +426,15 @@ public abstract class AbstractSAMLServletsAdapterTest extends AbstractServletsAd
     @Test
     public void salesPostSigPersistentTest() {
         salesPostSigPersistentServletPage.navigateTo();
-        testRealmSAMLLoginPage.form().login(bburkeUser);
+        testRealmSAMLPostLoginPage.form().login(bburkeUser);
         assertFalse(driver.getPageSource().contains("bburke"));
         assertTrue(driver.getPageSource().contains("principal=G-"));
 
         salesPostSigPersistentServletPage.logout();
         salesPostSigPersistentServletPage.navigateTo();
-        assertCurrentUrlStartsWith(testRealmSAMLLoginPage);
+        assertCurrentUrlStartsWith(testRealmSAMLPostLoginPage);
 
-        testRealmSAMLLoginPage.form().login("unauthorized", "password");
+        testRealmSAMLPostLoginPage.form().login("unauthorized", "password");
         assertFalse(driver.getPageSource().contains("principal="));
         //Different 403 status page on EAP and Wildfly
         assertTrue(driver.getPageSource().contains("Forbidden") || driver.getPageSource().contains("Status 403"));
@@ -435,15 +444,15 @@ public abstract class AbstractSAMLServletsAdapterTest extends AbstractServletsAd
     @Test
     public void salesPostSigTransientTest() {
         salesPostSigTransientServletPage.navigateTo();
-        testRealmSAMLLoginPage.form().login(bburkeUser);
+        testRealmSAMLPostLoginPage.form().login(bburkeUser);
         assertFalse(driver.getPageSource().contains("bburke"));
         assertTrue(driver.getPageSource().contains("principal=G-"));
 
         salesPostSigTransientServletPage.logout();
         salesPostSigTransientServletPage.navigateTo();
-        assertCurrentUrlStartsWith(testRealmSAMLLoginPage);
+        assertCurrentUrlStartsWith(testRealmSAMLPostLoginPage);
 
-        testRealmSAMLLoginPage.form().login("unauthorized", "password");
+        testRealmSAMLPostLoginPage.form().login("unauthorized", "password");
         assertFalse(driver.getPageSource().contains("principal="));
         //Different 403 status page on EAP and Wildfly
         assertTrue(driver.getPageSource().contains("Forbidden") || driver.getPageSource().contains("Status 403"));

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/admin/authentication/AbstractAuthenticationTest.java

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.keycloak.testsuite.authentication;
+package org.keycloak.testsuite.admin.authentication;
 
 import org.junit.Assert;
 import org.junit.Before;

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/admin/authentication/ExecutionTest.java

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.keycloak.testsuite.authentication;
+package org.keycloak.testsuite.admin.authentication;
 
 import org.junit.Assert;
 import org.junit.Test;

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/admin/authentication/FlowTest.java

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.keycloak.testsuite.authentication;
+package org.keycloak.testsuite.admin.authentication;
 
 import org.junit.Assert;
 import org.junit.Test;

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/admin/authentication/InitialFlowsTest.java

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.keycloak.testsuite.authentication;
+package org.keycloak.testsuite.admin.authentication;
 
 import org.junit.Assert;
 import org.junit.Test;

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/admin/authentication/InitialProvidersTest.java

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.keycloak.testsuite.authentication;
+package org.keycloak.testsuite.admin.authentication;
 
 import org.junit.Assert;
 import org.junit.Test;

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/admin/authentication/ShiftExecutionTest.java

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.keycloak.testsuite.authentication;
+package org.keycloak.testsuite.admin.authentication;
 
 import org.junit.Assert;
 import org.junit.Test;

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/admin/event/AbstractEventTest.java

@@ -15,7 +15,7 @@
  * the License.
  */
 
-package org.keycloak.testsuite.event;
+package org.keycloak.testsuite.admin.event;
 
 import java.util.Collections;
 import org.junit.Before;

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/admin/event/AdminEventTest.java

@@ -15,7 +15,7 @@
  * the License.
  */
 
-package org.keycloak.testsuite.event;
+package org.keycloak.testsuite.admin.event;
 
 import java.util.Arrays;
 import java.util.Collections;

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/admin/event/EventConfigTest.java

@@ -15,7 +15,7 @@
  * the License.
  */
 
-package org.keycloak.testsuite.event;
+package org.keycloak.testsuite.admin.event;
 
 import java.util.Arrays;
 import java.util.Collections;

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/admin/event/LoginEventsTest.java

@@ -15,7 +15,7 @@
  * the License.
  */
 
-package org.keycloak.testsuite.event;
+package org.keycloak.testsuite.admin.event;
 
 import java.util.Arrays;
 import java.util.List;

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/admin/partialimport/PartialImportTest.java

@@ -0,0 +1,464 @@
+/*
+ * Copyright 2016 Red Hat Inc. and/or its affiliates and other contributors
+ * as indicated by the @author tags. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.keycloak.testsuite.admin.partialimport;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import javax.ws.rs.ProcessingException;
+import javax.ws.rs.core.Response;
+import org.junit.Before;
+import org.junit.Test;
+import org.keycloak.admin.client.resource.ClientResource;
+import org.keycloak.admin.client.resource.IdentityProviderResource;
+import org.keycloak.admin.client.resource.RoleResource;
+import org.keycloak.admin.client.resource.UserResource;
+import org.keycloak.partialimport.PartialImportResult;
+import org.keycloak.partialimport.PartialImportResults;
+import org.keycloak.representations.idm.ClientRepresentation;
+import org.keycloak.representations.idm.IdentityProviderRepresentation;
+import org.keycloak.representations.idm.PartialImportRepresentation;
+import org.keycloak.representations.idm.UserRepresentation;
+import org.keycloak.testsuite.AbstractAuthTest;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import org.keycloak.representations.idm.PartialImportRepresentation.Policy;
+import org.keycloak.representations.idm.RoleRepresentation;
+import org.keycloak.representations.idm.RolesRepresentation;
+import org.keycloak.testsuite.admin.ApiUtil;
+
+/**
+ * Tests for the partial import endpoint in admin client.  Also tests the
+ * server side functionality of each resource along with "fail, skip, overwrite"
+ * functions.
+ *
+ * @author Stan Silvert ssilvert@redhat.com (C) 2016 Red Hat Inc.
+ */
+public class PartialImportTest extends AbstractAuthTest {
+
+    private static final int NUM_RESOURCE_TYPES = 5;
+    private static final String CLIENT_ROLES_CLIENT = "clientRolesClient";
+    private static final String USER_PREFIX = "user";
+    private static final String CLIENT_PREFIX = "client";
+    private static final String REALM_ROLE_PREFIX = "realmRole";
+    private static final String CLIENT_ROLE_PREFIX = "clientRole";
+    private static final String[] IDP_ALIASES = {"twitter", "github", "facebook", "google", "linkedin", "microsoft", "stackoverflow"};
+    private static final int NUM_ENTITIES = IDP_ALIASES.length;
+
+    private PartialImportRepresentation piRep;
+
+    @Before
+    public void init() {
+        piRep = new PartialImportRepresentation();
+    }
+
+    @Before
+    public void createClientForClientRoles() {
+        ClientRepresentation client = new ClientRepresentation();
+        client.setClientId(CLIENT_ROLES_CLIENT);
+        client.setName(CLIENT_ROLES_CLIENT);
+        client.setRootUrl("foo");
+        client.setProtocol("openid-connect");
+        Response resp = testRealmResource().clients().create(client);
+
+        // for some reason, findAll() will later fail unless readEntity is called here
+        resp.readEntity(String.class);
+        //testRealmResource().clients().findAll();
+    }
+
+    @Before
+    public void removeUsers() {
+        List<UserRepresentation> toRemove = testRealmResource().users().search(USER_PREFIX, 0, NUM_ENTITIES);
+        for (UserRepresentation user : toRemove) {
+            testRealmResource().users().get(user.getId()).remove();
+        }
+    }
+
+    @Before
+    public void removeClients() {
+        List<ClientRepresentation> toRemove = testRealmResource().clients().findAll();
+        for (ClientRepresentation client : toRemove) {
+            if (client.getName().startsWith(CLIENT_PREFIX)) {
+                testRealmResource().clients().get(client.getId()).remove();
+            }
+        }
+    }
+
+    @Before
+    public void removeProviders() {
+        List<IdentityProviderRepresentation> toRemove = testRealmResource().identityProviders().findAll();
+        for (IdentityProviderRepresentation idp : toRemove) {
+            testRealmResource().identityProviders().get(idp.getInternalId()).remove();
+        }
+    }
+
+    @Before
+    public void removeRealmRoles() {
+        List<RoleRepresentation> toRemove = testRealmResource().roles().list();
+        for (RoleRepresentation role : toRemove) {
+            if (role.getName().startsWith(REALM_ROLE_PREFIX)) {
+                testRealmResource().roles().get(role.getName()).remove();
+            }
+        }
+    }
+
+    @Before
+    public void removeClientRoles() {
+        List<RoleRepresentation> toRemove = clientRolesClient().roles().list();
+        for (RoleRepresentation role : toRemove) {
+            if (role.getName().startsWith(CLIENT_ROLE_PREFIX)) {
+                testRealmResource().clients().get(CLIENT_ROLES_CLIENT).roles().get(role.getName()).remove();
+            }
+        }
+    }
+
+    private ClientResource clientRolesClient() {
+        return ApiUtil.findClientResourceByName(testRealmResource(), CLIENT_ROLES_CLIENT);
+    }
+
+    private void setFail() {
+        piRep.setIfResourceExists(Policy.FAIL.toString());
+    }
+
+    private void setSkip() {
+        piRep.setIfResourceExists(Policy.SKIP.toString());
+    }
+
+    private void setOverwrite() {
+        piRep.setIfResourceExists(Policy.OVERWRITE.toString());
+    }
+
+    private PartialImportResults doImport() {
+        Response response = testRealmResource().partialImport(piRep);
+        return response.readEntity(PartialImportResults.class);
+    }
+
+    private void addUsers() {
+        List<UserRepresentation> users = new ArrayList<>();
+
+        for (int i = 0; i < NUM_ENTITIES; i++) {
+            UserRepresentation user = createUserRepresentation(USER_PREFIX + i, USER_PREFIX + i + "@foo.com", "foo", "bar", true);
+            users.add(user);
+        }
+
+        piRep.setUsers(users);
+    }
+
+    private void addClients() {
+        List<ClientRepresentation> clients = new ArrayList<>();
+
+        for (int i = 0; i < NUM_ENTITIES; i++) {
+            ClientRepresentation client = new ClientRepresentation();
+            client.setClientId(CLIENT_PREFIX + i);
+            client.setName(CLIENT_PREFIX + i);
+            client.setRootUrl("foo");
+            clients.add(client);
+        }
+
+        piRep.setClients(clients);
+    }
+
+    private void addProviders() {
+        List<IdentityProviderRepresentation> providers = new ArrayList<>();
+
+        for (String alias : IDP_ALIASES) {
+            IdentityProviderRepresentation idpRep = new IdentityProviderRepresentation();
+            idpRep.setAlias(alias);
+            idpRep.setProviderId(alias);
+            idpRep.setEnabled(true);
+            idpRep.setAuthenticateByDefault(false);
+            idpRep.setFirstBrokerLoginFlowAlias("first broker login");
+
+            Map<String, String> config = new HashMap<>();
+            config.put("clientSecret", "secret");
+            config.put("clientId", alias);
+            idpRep.setConfig(config);
+            providers.add(idpRep);
+        }
+
+        piRep.setIdentityProviders(providers);
+    }
+
+    private List<RoleRepresentation> makeRoles(String prefix) {
+        List<RoleRepresentation> roles = new ArrayList<>();
+
+        for (int i = 0; i < NUM_ENTITIES; i++) {
+            RoleRepresentation role = new RoleRepresentation();
+            role.setName(prefix + i);
+            roles.add(role);
+        }
+
+        return roles;
+    }
+
+    private void addRealmRoles() {
+        RolesRepresentation roles = piRep.getRoles();
+        if (roles == null) roles = new RolesRepresentation();
+        roles.setRealm(makeRoles(REALM_ROLE_PREFIX));
+        piRep.setRoles(roles);
+    }
+
+    private void addClientRoles() {
+        RolesRepresentation roles = piRep.getRoles();
+        if (roles == null) roles = new RolesRepresentation();
+        Map<String, List<RoleRepresentation>> clientRolesMap = new HashMap<>();
+        clientRolesMap.put(CLIENT_ROLES_CLIENT, makeRoles(CLIENT_ROLE_PREFIX));
+        roles.setClient(clientRolesMap);
+        piRep.setRoles(roles);
+    }
+
+    @Test
+    public void testAddUsers() {
+        setFail();
+        addUsers();
+
+        PartialImportResults results = doImport();
+        assertEquals(NUM_ENTITIES, results.getAdded());
+
+        for (PartialImportResult result : results.getResults()) {
+            String id = result.getId();
+            UserResource userRsc = testRealmResource().users().get(id);
+            UserRepresentation user = userRsc.toRepresentation();
+            assertTrue(user.getUsername().startsWith(USER_PREFIX));
+        }
+    }
+
+    @Test
+    public void testAddClients() {
+        setFail();
+        addClients();
+
+        PartialImportResults results = doImport();
+        assertEquals(NUM_ENTITIES, results.getAdded());
+
+        for (PartialImportResult result : results.getResults()) {
+            String id = result.getId();
+            ClientResource clientRsc = testRealmResource().clients().get(id);
+            ClientRepresentation client = clientRsc.toRepresentation();
+            assertTrue(client.getName().startsWith(CLIENT_PREFIX));
+        }
+    }
+
+    @Test
+    public void testAddProviders() {
+        setFail();
+        addProviders();
+
+        PartialImportResults results = doImport();
+        assertEquals(IDP_ALIASES.length, results.getAdded());
+
+        for (PartialImportResult result : results.getResults()) {
+            String id = result.getId();
+            IdentityProviderResource idpRsc = testRealmResource().identityProviders().get(id);
+            IdentityProviderRepresentation idp = idpRsc.toRepresentation();
+            Map<String, String> config = idp.getConfig();
+            assertTrue(Arrays.asList(IDP_ALIASES).contains(config.get("clientId")));
+        }
+    }
+
+    @Test
+    public void testAddRealmRoles() {
+        setFail();
+        addRealmRoles();
+
+        PartialImportResults results = doImport();
+        assertEquals(NUM_ENTITIES, results.getAdded());
+
+        for (PartialImportResult result : results.getResults()) {
+            String name = result.getResourceName();
+            RoleResource roleRsc = testRealmResource().roles().get(name);
+            RoleRepresentation role = roleRsc.toRepresentation();
+            assertTrue(role.getName().startsWith(REALM_ROLE_PREFIX));
+        }
+    }
+
+    @Test
+    public void testAddClientRoles() {
+        setFail();
+        addClientRoles();
+
+        PartialImportResults results = doImport();
+        assertEquals(NUM_ENTITIES, results.getAdded());
+
+        List<RoleRepresentation> clientRoles = clientRolesClient().roles().list();
+        assertEquals(NUM_ENTITIES, clientRoles.size());
+
+        for (RoleRepresentation roleRep : clientRoles) {
+            assertTrue(roleRep.getName().startsWith(CLIENT_ROLE_PREFIX));
+        }
+    }
+
+    private void testFail() {
+        setFail();
+        PartialImportResults results = doImport();
+        assertNull(results.getErrorMessage());
+        results = doImport(); // second time should fail
+        assertNotNull(results.getErrorMessage());
+    }
+
+    @Test
+    public void testAddUsersFail() {
+        addUsers();
+        testFail();
+    }
+
+    @Test
+    public void testAddClientsFail() {
+        addClients();
+        testFail();
+    }
+
+    @Test
+    public void testAddProvidersFail() {
+        addProviders();
+        testFail();
+    }
+
+    @Test
+    public void testAddRealmRolesFail() {
+        addRealmRoles();
+        testFail();
+    }
+
+    @Test
+    public void testAddClientRolesFail() {
+        addClientRoles();
+        testFail();
+    }
+
+    private void testSkip() {
+        setSkip();
+        PartialImportResults results = doImport();
+        assertEquals(NUM_ENTITIES, results.getAdded());
+
+        results = doImport();
+        assertEquals(NUM_ENTITIES, results.getSkipped());
+    }
+
+    @Test
+    public void testAddUsersSkip() {
+        addUsers();
+        testSkip();
+    }
+
+    @Test
+    public void testAddClientsSkip() {
+        addClients();
+        testSkip();
+    }
+
+    @Test
+    public void testAddProvidersSkip() {
+        addProviders();
+        testSkip();
+    }
+
+    @Test
+    public void testAddRealmRolesSkip() {
+        addRealmRoles();
+        testSkip();
+    }
+
+    @Test
+    public void testAddClientRolesSkip() {
+        addClientRoles();
+        testSkip();
+    }
+
+    private void testOverwrite() {
+        setOverwrite();
+        PartialImportResults results = doImport();
+        assertEquals(NUM_ENTITIES, results.getAdded());
+
+        results = doImport();
+        assertEquals(NUM_ENTITIES, results.getOverwritten());
+    }
+
+    @Test
+    public void testAddUsersOverwrite() {
+        addUsers();
+        testOverwrite();
+    }
+
+    @Test
+    public void testAddClientsOverwrite() {
+        addClients();
+        testOverwrite();
+    }
+
+    @Test
+    public void testAddProvidersOverwrite() {
+        addProviders();
+        testOverwrite();
+    }
+
+    @Test
+    public void testAddRealmRolesOverwrite() {
+        addRealmRoles();
+        testOverwrite();
+    }
+
+    @Test
+    public void testAddClientRolesOverwrite() {
+        addClientRoles();
+        testOverwrite();
+    }
+
+
+    private void importEverything() {
+        addUsers();
+        addClients();
+        addProviders();
+        addRealmRoles();
+        addClientRoles();
+
+        PartialImportResults results = doImport();
+        assertNull(results.getErrorMessage());
+        assertEquals(NUM_ENTITIES * NUM_RESOURCE_TYPES, results.getAdded());
+    }
+
+    @Test
+    public void testEverythingFail() {
+        setFail();
+        importEverything();
+        PartialImportResults results = doImport(); // second import will fail because not allowed to skip or overwrite
+        assertNotNull(results.getErrorMessage());
+    }
+
+    @Test
+    public void testEverythingSkip() {
+        setSkip();
+        importEverything();
+        PartialImportResults results = doImport();
+        assertEquals(NUM_ENTITIES * NUM_RESOURCE_TYPES, results.getSkipped());
+    }
+
+    @Test
+    public void testEverythingOverwrite() {
+        setOverwrite();
+        importEverything();
+        PartialImportResults results = doImport();
+        assertEquals(NUM_ENTITIES * NUM_RESOURCE_TYPES, results.getOverwritten());
+    }
+
+}

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractClusterTest.java

@@ -1,19 +1,22 @@
 package org.keycloak.testsuite.cluster;
 
-import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import org.jboss.arquillian.container.test.api.ContainerController;
-import org.jboss.arquillian.graphene.page.Page;
 import org.jboss.arquillian.test.api.ArquillianResource;
 import static org.junit.Assert.assertTrue;
+import org.junit.Before;
 import org.keycloak.admin.client.Keycloak;
 import org.keycloak.models.Constants;
 import org.keycloak.representations.idm.RealmRepresentation;
 import org.keycloak.testsuite.AbstractKeycloakTest;
 import org.keycloak.testsuite.arquillian.ContainerInfo;
-import org.keycloak.testsuite.auth.page.AuthRealm;
 import static org.keycloak.testsuite.auth.page.AuthRealm.ADMIN;
 import static org.keycloak.testsuite.auth.page.AuthRealm.MASTER;
+import static org.keycloak.testsuite.util.WaitUtils.pause;
 
 /**
  *
@@ -24,52 +27,103 @@ public abstract class AbstractClusterTest extends AbstractKeycloakTest {
     @ArquillianResource
     protected ContainerController controller;
 
-    protected List<Keycloak> backendAdminClients = new ArrayList<>();
+    protected Map<ContainerInfo, Keycloak> backendAdminClients = new HashMap<>();
 
-    public void startBackendNodes(int count) {
-        if (count < 0 || count > 10) {
-            throw new IllegalArgumentException();
+    private int currentFailNodeIndex = 0;
+
+    public int getClusterSize() {
+        return suiteContext.getAuthServerBackendsInfo().size();
+    }
+
+    protected void iterateCurrentFailNode() {
+        currentFailNodeIndex++;
+        if (currentFailNodeIndex >= getClusterSize()) {
+            currentFailNodeIndex = 0;
         }
-        assertTrue(suiteContext.getAuthServerBackendsInfo().size() >= count);
-        for (int i = 0; i < count; i++) {
+        logFailoverSetup();
+    }
 
-            ContainerInfo backendNode = suiteContext.getAuthServerBackendsInfo().get(i);
+    protected ContainerInfo getCurrentFailNode() {
+        return backendNode(currentFailNodeIndex);
+    }
 
-            controller.start(backendNode.getQualifier());
-            assertTrue(controller.isStarted(backendNode.getQualifier()));
+    protected Set<ContainerInfo> getCurrentSurvivorNodes() {
+        Set<ContainerInfo> survivors = new HashSet<>(suiteContext.getAuthServerBackendsInfo());
+        survivors.remove(getCurrentFailNode());
+        return survivors;
+    }
 
-            backendAdminClients.add(createAdminClientFor(backendNode));
+    protected void logFailoverSetup() {
+        log.info("Current failover setup");
+        boolean started = controller.isStarted(getCurrentFailNode().getQualifier());
+        log.info("Fail node: " + getCurrentFailNode() + (started ? "" : " (stopped)"));
+        for (ContainerInfo survivor : getCurrentSurvivorNodes()) {
+            started = controller.isStarted(survivor.getQualifier());
+            log.info("Survivor:  " + survivor + (started ? "" : " (stopped)"));
         }
     }
 
-    protected Keycloak createAdminClientFor(ContainerInfo backendNode) {
-        log.info("Initializing admin client for " + backendNode.getContextRoot() + "/auth");
-        return Keycloak.getInstance(backendNode.getContextRoot() + "/auth",
-                MASTER, ADMIN, ADMIN, Constants.ADMIN_CLI_CLIENT_ID);
+    public void failure() {
+        log.info("Simulating failure");
+        killBackendNode(getCurrentFailNode());
+    }
+
+    public void failback() {
+        log.info("Bringing all backend nodes online");
+        for (ContainerInfo node : suiteContext.getAuthServerBackendsInfo()) {
+            startBackendNode(node);
+        }
+    }
+
+    protected ContainerInfo frontendNode() {
+        return suiteContext.getAuthServerInfo();
     }
 
     protected ContainerInfo backendNode(int i) {
         return suiteContext.getAuthServerBackendsInfo().get(i);
     }
 
-    protected void startBackendNode(int i) {
-        String container = backendNode(i).getQualifier();
-        if (!controller.isStarted(container)) {
-            controller.start(container);
-            backendAdminClients.set(i, createAdminClientFor(backendNode(i)));
+    protected void startBackendNode(ContainerInfo node) {
+        if (!controller.isStarted(node.getQualifier())) {
+            log.info("Starting backend node: " + node);
+            controller.start(node.getQualifier());
+            assertTrue(controller.isStarted(node.getQualifier()));
+        }
+        log.info("Backend node " + node + " is started");
+        if (!backendAdminClients.containsKey(node)) {
+            backendAdminClients.put(node, createAdminClientFor(node));
         }
     }
 
-    protected void killBackendNode(int i) {
-        backendAdminClients.get(i).close();
-        controller.kill(backendNode(i).getQualifier());
+    protected Keycloak createAdminClientFor(ContainerInfo node) {
+        log.info("Initializing admin client for " + node.getContextRoot() + "/auth");
+        return Keycloak.getInstance(node.getContextRoot() + "/auth",
+                MASTER, ADMIN, ADMIN, Constants.ADMIN_CLI_CLIENT_ID);
     }
 
-    protected void listRealms(int i) {
-        log.info(String.format("Node %s: AccessTokenString: %s", i + 1, backendAdminClients.get(i).tokenManager().getAccessTokenString()));
-        for (RealmRepresentation r : backendAdminClients.get(i).realms().findAll()) {
-            log.info(String.format("Node %s: Realm: %s, Id: %s", i + 1, r.getRealm(), r.getId()));
-        }
+    protected void killBackendNode(ContainerInfo node) {
+        backendAdminClients.get(node).close();
+        backendAdminClients.remove(node);
+        log.info("Killing backend node: " + node);
+        controller.kill(node.getQualifier());
     }
-    
+
+    protected Keycloak getAdminClientFor(ContainerInfo node) {
+        return node.equals(suiteContext.getAuthServerInfo())
+                ? adminClient // frontend client
+                : backendAdminClients.get(node);
+    }
+
+    @Before
+    public void beforeClusterTest() {
+        failback();
+        logFailoverSetup();
+        pause(3000);
+    }
+
+    @Override
+    public void addTestRealms(List<RealmRepresentation> testRealms) {
+        // no test realms will be created by the default 
+    }
+
 }

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractInvalidationClusterTest.java

@@ -0,0 +1,165 @@
+package org.keycloak.testsuite.cluster;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.builder.EqualsBuilder;
+import org.apache.commons.lang.builder.ReflectionToStringBuilder;
+import org.apache.commons.lang.builder.ToStringStyle;
+import static org.junit.Assert.assertFalse;
+import org.junit.Test;
+import org.keycloak.representations.idm.RealmRepresentation;
+import org.keycloak.testsuite.arquillian.ContainerInfo;
+
+/**
+ *
+ * @author tkyjovsk
+ * @param <T> entity representation
+ * @param <TR> entity resource
+ */
+public abstract class AbstractInvalidationClusterTest<T, TR> extends AbstractClusterTest {
+
+    protected RealmRepresentation createTestRealmRepresentation() {
+        RealmRepresentation testRealm = new RealmRepresentation();
+        testRealm.setRealm("test_" + RandomStringUtils.randomAlphabetic(5));
+        testRealm.setEnabled(true);
+        return testRealm;
+    }
+
+    protected abstract T createTestEntityRepresentation();
+
+    @Test
+    public void crudWithoutFailover() {
+        crud(false);
+    }
+
+    @Test
+    public void crudWithFailover() {
+        crud(true);
+    }
+
+    public void crud(boolean backendFailover) {
+        T testEntity = createTestEntityRepresentation();
+
+        // CREATE 
+        testEntity = createEntityOnCurrentFailNode(testEntity);
+
+        if (backendFailover) {
+            failure();
+        }
+
+        assertEntityOnSurvivorNodesEqualsTo(testEntity);
+
+        failback();
+        iterateCurrentFailNode();
+
+        // UPDATE(s)
+        testEntity = testEntityUpdates(testEntity, backendFailover);
+
+        // DELETE 
+        deleteEntityOnCurrentFailNode(testEntity);
+
+        if (backendFailover) {
+            failure();
+        }
+
+        assertEntityOnSurvivorNodesIsDeleted(testEntity);
+    }
+
+    protected abstract TR entityResource(T testEntity, ContainerInfo node);
+
+    protected abstract TR entityResource(String idOrName, ContainerInfo node);
+    
+    protected abstract T createEntity(T testEntity, ContainerInfo node);
+
+    protected abstract T readEntity(T entity, ContainerInfo node);
+
+    protected abstract T updateEntity(T entity, ContainerInfo node);
+
+    protected abstract void deleteEntity(T testEntity, ContainerInfo node);
+
+    protected TR entityResourceOnCurrentFailNode(T testEntity) {
+        return entityResource(testEntity, getCurrentFailNode());
+    }
+
+    protected String getEntityType(T entity) {
+        return entity.getClass().getSimpleName().replace("Representation", "");
+    }
+
+    protected T createEntityOnCurrentFailNode(T entity) {
+        log.info("Creating " + getEntityType(entity) + " on " + getCurrentFailNode());
+        return createEntity(entity, getCurrentFailNode());
+    }
+
+    protected T readEntityOnCurrentFailNode(T entity) {
+        log.debug("Reading " + getEntityType(entity) + " on " + getCurrentFailNode());
+        return readEntity(entity, getCurrentFailNode());
+    }
+
+    protected T updateEntityOnCurrentFailNode(T entity) {
+        return updateEntityOnCurrentFailNode(entity, "");
+    }
+
+    protected T updateEntityOnCurrentFailNode(T entity, String updateType) {
+        log.info("Updating " + getEntityType(entity) + " " + updateType + " on " + getCurrentFailNode());
+        return updateEntity(entity, getCurrentFailNode());
+    }
+
+    protected void deleteEntityOnCurrentFailNode(T entity) {
+        log.info("Creating " + getEntityType(entity) + " on " + getCurrentFailNode());
+        deleteEntity(entity, getCurrentFailNode());
+    }
+
+    protected abstract T testEntityUpdates(T testEntity, boolean backendFailover);
+
+    protected void verifyEntityUpdateDuringFailover(T testEntity, boolean backendFailover) {
+        if (backendFailover) {
+            failure();
+        }
+
+        assertEntityOnSurvivorNodesEqualsTo(testEntity);
+
+        failback();
+        iterateCurrentFailNode();
+    }
+
+    protected List<String> excludedComparisonFields = new ArrayList<>();
+
+    protected void assertEntityOnSurvivorNodesEqualsTo(T testEntityOnFailNode) {
+        boolean entityDiffers = false;
+        for (ContainerInfo survivorNode : getCurrentSurvivorNodes()) {
+            T testEntityOnSurvivorNode = readEntity(testEntityOnFailNode, survivorNode);
+            if (EqualsBuilder.reflectionEquals(testEntityOnSurvivorNode, testEntityOnFailNode, excludedComparisonFields)) {
+                log.info(String.format("Verification of %s on survivor %s PASSED", getEntityType(testEntityOnFailNode), survivorNode));
+            } else {
+                entityDiffers = true;
+                log.error(String.format("Verification of %s on survivor %s FAILED", getEntityType(testEntityOnFailNode), survivorNode));
+                String tf = ReflectionToStringBuilder.reflectionToString(testEntityOnFailNode, ToStringStyle.SHORT_PREFIX_STYLE);
+                String ts = ReflectionToStringBuilder.reflectionToString(testEntityOnSurvivorNode, ToStringStyle.SHORT_PREFIX_STYLE);
+                log.error(String.format(
+                        "\nEntity on fail node: \n%s\n"
+                        + "\nEntity on survivor node: \n%s\n"
+                        + "\nDifference: \n%s\n",
+                        tf, ts, StringUtils.difference(tf, ts)));
+            }
+        }
+        assertFalse(entityDiffers);
+    }
+
+    private void assertEntityOnSurvivorNodesIsDeleted(T testEntityOnFailNode) {
+        // check if deleted from all survivor nodes
+        boolean entityExists = false;
+        for (ContainerInfo survivorNode : getCurrentSurvivorNodes()) {
+            T testEntityOnSurvivorNode = readEntity(testEntityOnFailNode, survivorNode);
+            if (testEntityOnSurvivorNode == null) {
+                log.info(String.format("Verification of %s deletion on survivor %s PASSED", getEntityType(testEntityOnFailNode), survivorNode));
+            } else {
+                entityExists = true;
+                log.error(String.format("Verification of %s deletion on survivor %s FAILED", getEntityType(testEntityOnFailNode), survivorNode));
+            }
+        }
+        assertFalse(entityExists);
+    }
+
+}

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractInvalidationClusterTestWithTestRealm.java

@@ -0,0 +1,26 @@
+package org.keycloak.testsuite.cluster;
+
+import org.junit.Before;
+import org.keycloak.representations.idm.RealmRepresentation;
+import org.keycloak.testsuite.arquillian.ContainerInfo;
+
+/**
+ *
+ * @author tkyjovsk
+ */
+public abstract class AbstractInvalidationClusterTestWithTestRealm<T, TR> extends AbstractInvalidationClusterTest<T, TR> {
+
+    protected String testRealmName = null;
+    
+    @Before
+    public void createTestRealm() {
+        createTestRealm(frontendNode());
+    }
+    
+    protected void createTestRealm(ContainerInfo node) {
+        RealmRepresentation r = createTestRealmRepresentation();
+        getAdminClientFor(node).realms().create(r);
+        testRealmName = r.getRealm();
+    }
+    
+}

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractTwoNodeClusterTest.java

@@ -1,57 +0,0 @@
-package org.keycloak.testsuite.cluster;
-
-import org.junit.Before;
-import org.keycloak.admin.client.Keycloak;
-import org.keycloak.testsuite.arquillian.ContainerInfo;
-import static org.keycloak.testsuite.util.WaitUtils.pause;
-
-/**
- *
- * @author tkyjovsk
- */
-public abstract class AbstractTwoNodeClusterTest extends AbstractClusterTest {
-
-    @Before
-    public void beforeTwoNodeClusterTest() {
-        startBackendNodes(2);
-        pause(3000);
-    }
-
-    protected ContainerInfo backend1Info() {
-        return backendNode(0);
-    }
-
-    protected ContainerInfo backend2Info() {
-        return backendNode(1);
-    }
-
-    protected Keycloak backend1AdminClient() {
-        return backendAdminClients.get(0);
-    }
-
-    protected Keycloak backend2AdminClient() {
-        return backendAdminClients.get(1);
-    }
-
-    protected void startBackend1() {
-        startBackendNode(0);
-    }
-
-    protected void startBackend2() {
-        startBackendNode(1);
-    }
-
-    protected void failback() {
-        startBackend1();
-        startBackend2();
-    }
-
-    protected void killBackend1() {
-        killBackendNode(0);
-    }
-
-    protected void killBackend2() {
-        killBackendNode(1);
-    }
-
-}

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/ClientInvalidationClusterTest.java

@@ -0,0 +1,96 @@
+package org.keycloak.testsuite.cluster;
+
+import javax.ws.rs.NotFoundException;
+import javax.ws.rs.core.Response;
+import org.apache.commons.lang.RandomStringUtils;
+import static org.junit.Assert.assertNull;
+import org.junit.Before;
+import org.keycloak.admin.client.resource.ClientResource;
+import org.keycloak.admin.client.resource.ClientsResource;
+import org.keycloak.representations.idm.ClientRepresentation;
+import org.keycloak.testsuite.admin.ApiUtil;
+import org.keycloak.testsuite.arquillian.ContainerInfo;
+
+/**
+ *
+ * @author tkyjovsk
+ */
+public class ClientInvalidationClusterTest extends AbstractInvalidationClusterTestWithTestRealm<ClientRepresentation, ClientResource> {
+
+    @Before
+    public void setExcludedComparisonFields() {
+        excludedComparisonFields.add("protocolMappers");
+    }
+
+    @Override
+    protected ClientRepresentation createTestEntityRepresentation() {
+        ClientRepresentation client = new ClientRepresentation();
+        String s = RandomStringUtils.randomAlphabetic(5);
+        client.setClientId("client_" + s);
+        client.setName("name_" + s);
+        return client;
+    }
+
+    protected ClientsResource clients(ContainerInfo node) {
+        return getAdminClientFor(node).realm(testRealmName).clients();
+    }
+
+    @Override
+    protected ClientResource entityResource(ClientRepresentation client, ContainerInfo node) {
+        return entityResource(client.getId(), node);
+    }
+
+    @Override
+    protected ClientResource entityResource(String id, ContainerInfo node) {
+        return clients(node).get(id);
+    }
+    
+    @Override
+    protected ClientRepresentation createEntity(ClientRepresentation client, ContainerInfo node) {
+        Response response = clients(node).create(client);
+        String id = ApiUtil.getCreatedId(response);
+        response.close();
+        client.setId(id);
+        return readEntity(client, node);
+    }
+
+    @Override
+    protected ClientRepresentation readEntity(ClientRepresentation client, ContainerInfo node) {
+        ClientRepresentation u = null;
+        try {
+            u = entityResource(client, node).toRepresentation();
+        } catch (NotFoundException nfe) {
+            // expected when client doesn't exist
+        }
+        return u;
+    }
+
+    @Override
+    protected ClientRepresentation updateEntity(ClientRepresentation client, ContainerInfo node) {
+        entityResource(client, node).update(client);
+        return readEntity(client, node);
+    }
+
+    @Override
+    protected void deleteEntity(ClientRepresentation client, ContainerInfo node) {
+        entityResource(client, node).remove();
+        assertNull(readEntity(client, node));
+    }
+
+    @Override
+    protected ClientRepresentation testEntityUpdates(ClientRepresentation client, boolean backendFailover) {
+
+        // clientId
+        client.setClientId(client.getClientId() + "_updated");
+        client = updateEntityOnCurrentFailNode(client, "clientId");
+        verifyEntityUpdateDuringFailover(client, backendFailover);
+
+        // name
+        client.setName(client.getName() + "_updated");
+        client = updateEntityOnCurrentFailNode(client, "name");
+        verifyEntityUpdateDuringFailover(client, backendFailover);
+
+        return client;
+    }
+
+}

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/EntityInvalidationClusterTest.java

@@ -1,109 +0,0 @@
-package org.keycloak.testsuite.cluster;
-
-import java.util.List;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import org.junit.Test;
-import org.keycloak.representations.idm.RealmRepresentation;
-import static org.keycloak.testsuite.auth.page.AuthRealm.TEST;
-
-/**
- *
- * @author tkyjovsk
- */
-public class EntityInvalidationClusterTest extends AbstractTwoNodeClusterTest {
-
-    @Override
-    public void addTestRealms(List<RealmRepresentation> testRealms) {
-    }
-
-    @Test
-    public void realmCRUDWithoutFailover() {
-        realmCRUD(TEST + "_wofo", false);
-    }
-
-    @Test
-    public void realmCRUDWithFailover() {
-        realmCRUD(TEST + "_wfo", true);
-    }
-
-    public void realmCRUD(String realm, boolean containerFailover) {
-        RealmRepresentation testRealm = new RealmRepresentation();
-        testRealm.setRealm(realm);
-        testRealm.setEnabled(true);
-
-        // CREATE on node1
-        backend1AdminClient().realms().create(testRealm);
-
-        // check if created on node1
-        RealmRepresentation testRealmOnBackend1 = backend1AdminClient().realms().realm(realm).toRepresentation();
-        assertEquals(testRealmOnBackend1.getRealm(), testRealm.getRealm());
-        if (containerFailover) {
-            killBackend1();
-        }
-
-        // check if created on node2
-        RealmRepresentation testRealmOnBackend2 = backend2AdminClient().realms().realm(realm).toRepresentation();
-        assertEquals(testRealmOnBackend1.getId(), testRealmOnBackend2.getId());
-        assertEquals(testRealmOnBackend1.getRealm(), testRealmOnBackend2.getRealm());
-
-        failback();
-
-        // UPDATE on node2
-        String realmUpdated = realm + "_updated";
-        testRealmOnBackend2.setRealm(realmUpdated);
-        backend2AdminClient().realms().realm(realm).update(testRealmOnBackend2);
-        if (containerFailover) {
-            killBackend2();
-        }
-        // check if updated on node1
-        testRealmOnBackend1 = backend1AdminClient().realms().realm(realmUpdated).toRepresentation();
-        assertEquals(testRealmOnBackend1.getId(), testRealmOnBackend2.getId());
-        assertEquals(testRealmOnBackend1.getRealm(), testRealmOnBackend2.getRealm());
-
-        failback();
-
-        // DELETE on node1
-        backend1AdminClient().realms().realm(realmUpdated).remove();
-        if (containerFailover) {
-            killBackend1();
-        }
-        // check if deleted on node2
-        boolean testRealmOnBackend2Exists = false;
-        for (RealmRepresentation realmOnBackend2 : backend2AdminClient().realms().findAll()) {
-            if (realmUpdated.equals(realmOnBackend2.getRealm())
-                    || testRealmOnBackend1.getId().equals(realmOnBackend2.getId())) {
-                testRealmOnBackend2Exists = true;
-                break;
-            }
-        }
-        assertFalse(testRealmOnBackend2Exists);
-    }
-
-    @Test
-    public void createRealmViaFrontend() {
-        String realm = TEST + "_fe";
-
-        RealmRepresentation testRealm = new RealmRepresentation();
-        testRealm.setRealm(realm);
-        testRealm.setEnabled(true);
-
-        // CREATE on frontend
-        adminClient.realms().create(testRealm);
-
-        // check if created on frontend
-        RealmRepresentation testRealmOnFrontend = adminClient.realms().realm(realm).toRepresentation();
-        assertEquals(testRealmOnFrontend.getRealm(), testRealm.getRealm());
-
-        // check if created on node1
-        RealmRepresentation testRealmOnBackend1 = backend1AdminClient().realms().realm(realm).toRepresentation();
-        assertEquals(testRealmOnBackend1.getId(), testRealmOnFrontend.getId());
-        assertEquals(testRealmOnBackend1.getRealm(), testRealmOnFrontend.getRealm());
-
-        // check if created on node2
-        RealmRepresentation testRealmOnBackend2 = backend2AdminClient().realms().realm(realm).toRepresentation();
-        assertEquals(testRealmOnBackend2.getId(), testRealmOnFrontend.getId());
-        assertEquals(testRealmOnBackend2.getRealm(), testRealmOnFrontend.getRealm());
-    }
-    
-}

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/GroupInvalidationClusterTest.java

@@ -0,0 +1,136 @@
+package org.keycloak.testsuite.cluster;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import javax.ws.rs.NotFoundException;
+import javax.ws.rs.core.Response;
+import org.apache.commons.lang.RandomStringUtils;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import org.junit.Before;
+import org.keycloak.admin.client.resource.GroupResource;
+import org.keycloak.admin.client.resource.GroupsResource;
+import org.keycloak.representations.idm.GroupRepresentation;
+import org.keycloak.testsuite.admin.ApiUtil;
+import org.keycloak.testsuite.arquillian.ContainerInfo;
+
+/**
+ *
+ * @author tkyjovsk
+ */
+public class GroupInvalidationClusterTest extends AbstractInvalidationClusterTestWithTestRealm<GroupRepresentation, GroupResource> {
+
+    @Before
+    public void setExcludedComparisonFields() {
+        excludedComparisonFields.add("subGroups");
+    }
+
+    @Override
+    protected GroupRepresentation createTestEntityRepresentation() {
+        GroupRepresentation group = new GroupRepresentation();
+        group.setName("group_" + RandomStringUtils.randomAlphabetic(5));
+        group.setAttributes(new HashMap<String, List<String>>());
+        group.getAttributes().put("attr1", Arrays.asList(new String[]{"attr1 value"}));
+        group.getAttributes().put("attr2", Arrays.asList(new String[]{"attr2 value", "attr2 value2"}));
+        return group;
+    }
+
+    protected GroupsResource groups(ContainerInfo node) {
+        return getAdminClientFor(node).realm(testRealmName).groups();
+    }
+
+    @Override
+    protected GroupResource entityResource(GroupRepresentation group, ContainerInfo node) {
+        return entityResource(group.getId(), node);
+    }
+
+    @Override
+    protected GroupResource entityResource(String id, ContainerInfo node) {
+        return groups(node).group(id);
+    }
+
+    @Override
+    protected GroupRepresentation createEntity(GroupRepresentation group, ContainerInfo node) {
+        Response response = groups(node).add(group);
+        String id = ApiUtil.getCreatedId(response);
+        response.close();
+        group.setId(id);
+        return readEntity(group, node);
+    }
+
+    @Override
+    protected GroupRepresentation readEntity(GroupRepresentation group, ContainerInfo node) {
+        GroupRepresentation u = null;
+        try {
+            u = entityResource(group, node).toRepresentation();
+        } catch (NotFoundException nfe) {
+            // expected when group doesn't exist
+        }
+        return u;
+    }
+
+    @Override
+    protected GroupRepresentation updateEntity(GroupRepresentation group, ContainerInfo node) {
+        entityResource(group, node).update(group);
+        return readEntity(group, node);
+    }
+
+    @Override
+    protected void deleteEntity(GroupRepresentation group, ContainerInfo node) {
+        entityResource(group, node).remove();
+        assertNull(readEntity(group, node));
+    }
+
+    @Override
+    protected GroupRepresentation testEntityUpdates(GroupRepresentation group, boolean backendFailover) {
+
+        // groupname
+        group.setName(group.getName() + "_updated");
+        group = updateEntityOnCurrentFailNode(group, "name");
+        verifyEntityUpdateDuringFailover(group, backendFailover);
+
+        // attributes - add new
+        group.getAttributes().put("attr3", Arrays.asList(new String[]{"attr3 value"}));
+        group = updateEntityOnCurrentFailNode(group, "attributes - adding");
+        verifyEntityUpdateDuringFailover(group, backendFailover);
+
+        // attributes - remove
+        group.getAttributes().remove("attr3");
+        group = updateEntityOnCurrentFailNode(group, "attributes - removing");
+        verifyEntityUpdateDuringFailover(group, backendFailover);
+
+        // attributes - update 1
+        group.getAttributes().get("attr1").set(0,
+                group.getAttributes().get("attr1").get(0) + " - updated");
+        group = updateEntityOnCurrentFailNode(group, "attributes");
+        verifyEntityUpdateDuringFailover(group, backendFailover);
+
+        // attributes - update 2
+        group.getAttributes().get("attr2").set(1,
+                group.getAttributes().get("attr2").get(1) + " - updated");
+        group = updateEntityOnCurrentFailNode(group, "attributes");
+        verifyEntityUpdateDuringFailover(group, backendFailover);
+
+        // move 
+        log.info("Updating Group parent on " + getCurrentFailNode());
+        GroupRepresentation parentGroup = new GroupRepresentation();
+        parentGroup.setName("parent");
+        parentGroup = createEntityOnCurrentFailNode(parentGroup);
+        assertEquals("/" + parentGroup.getName(), parentGroup.getPath());
+
+        Response r = entityResourceOnCurrentFailNode(parentGroup).subGroup(group);
+        r.close();
+        parentGroup = readEntityOnCurrentFailNode(parentGroup);
+        group = readEntityOnCurrentFailNode(group);
+
+        assertTrue(ApiUtil.groupContainsSubgroup(parentGroup, group));
+        assertEquals(parentGroup.getPath() + "/" + group.getName(), group.getPath());
+
+        verifyEntityUpdateDuringFailover(group, backendFailover);
+
+        return group;
+    }
+
+}

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/RealmInvalidationClusterTest.java

@@ -0,0 +1,109 @@
+package org.keycloak.testsuite.cluster;
+
+import javax.ws.rs.NotFoundException;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNull;
+import org.keycloak.admin.client.resource.RealmResource;
+import org.keycloak.admin.client.resource.RealmsResource;
+import org.keycloak.representations.idm.RealmRepresentation;
+import org.keycloak.testsuite.arquillian.ContainerInfo;
+
+/**
+ *
+ * @author tkyjovsk
+ */
+public class RealmInvalidationClusterTest extends AbstractInvalidationClusterTest<RealmRepresentation, RealmResource> {
+
+    @Override
+    protected RealmRepresentation createTestEntityRepresentation() {
+        return createTestRealmRepresentation();
+    }
+
+    protected RealmsResource realms(ContainerInfo node) {
+        return getAdminClientFor(node).realms();
+    }
+
+    @Override
+    protected RealmResource entityResource(RealmRepresentation realm, ContainerInfo node) {
+        return entityResource(realm.getRealm(), node);
+    }
+
+    @Override
+    protected RealmResource entityResource(String name, ContainerInfo node) {
+        return getAdminClientFor(node).realm(name);
+    }
+
+    @Override
+    protected RealmRepresentation createEntity(RealmRepresentation realm, ContainerInfo node) {
+        realms(node).create(realm);
+        return readEntity(realm, node);
+    }
+
+    @Override
+    protected RealmRepresentation readEntity(RealmRepresentation realm, ContainerInfo node) {
+        RealmRepresentation realmOnNode = null;
+        try {
+            realmOnNode = entityResource(realm, node).toRepresentation();
+        } catch (NotFoundException nfe) {
+            // expected if realm not found
+        }
+        return realmOnNode;
+    }
+
+    @Override
+    protected RealmRepresentation updateEntity(RealmRepresentation realm, ContainerInfo node) {
+        return updateEntity(realm.getRealm(), realm, node);
+    }
+
+    private RealmRepresentation updateEntity(String realmName, RealmRepresentation realm, ContainerInfo node) {
+        entityResource(realmName, node).update(realm);
+        return readEntity(realm, node);
+    }
+
+    @Override
+    protected void deleteEntity(RealmRepresentation realm, ContainerInfo node) {
+        entityResource(realm, node).remove();
+        // check if deleted
+        assertNull(readEntity(realm, node));
+    }
+
+    @Override
+    protected RealmRepresentation testEntityUpdates(RealmRepresentation realm, boolean backendFailover) {
+
+        // realm name
+        String originalName = realm.getRealm();
+        realm.setRealm(realm.getRealm() + "_updated");
+        realm = updateEntity(originalName, realm, getCurrentFailNode());
+        verifyEntityUpdateDuringFailover(realm, backendFailover);
+
+        // enabled
+        realm.setEnabled(!realm.isEnabled());
+        realm = updateEntityOnCurrentFailNode(realm, "enabled");
+        verifyEntityUpdateDuringFailover(realm, backendFailover);
+
+        // public key
+        realm.setPublicKey("GENERATE");
+        realm = updateEntityOnCurrentFailNode(realm, "public key");
+        assertNotEquals("GENERATE", realm.getPublicKey());
+        verifyEntityUpdateDuringFailover(realm, backendFailover);
+
+        // require ssl
+        realm.setSslRequired("all");
+        realm = updateEntityOnCurrentFailNode(realm, "require ssl");
+        verifyEntityUpdateDuringFailover(realm, backendFailover);
+
+        // brute force detection
+        realm.setBruteForceProtected(!realm.isBruteForceProtected());
+        realm = updateEntityOnCurrentFailNode(realm, "brute force");
+        verifyEntityUpdateDuringFailover(realm, backendFailover);
+
+        // brute force detection - failure factor
+        realm.setBruteForceProtected(true);
+        realm.setFailureFactor(realm.getFailureFactor() + 1);
+        realm = updateEntityOnCurrentFailNode(realm, "brute force failure factor");
+        verifyEntityUpdateDuringFailover(realm, backendFailover);
+
+        return realm;
+    }
+
+}

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/RoleInvalidationClusterTest.java

@@ -0,0 +1,86 @@
+package org.keycloak.testsuite.cluster;
+
+import javax.ws.rs.NotFoundException;
+import org.apache.commons.lang.RandomStringUtils;
+import static org.junit.Assert.assertNull;
+import org.keycloak.admin.client.resource.RoleResource;
+import org.keycloak.admin.client.resource.RolesResource;
+import org.keycloak.representations.idm.RoleRepresentation;
+import org.keycloak.testsuite.arquillian.ContainerInfo;
+
+/**
+ *
+ * @author tkyjovsk
+ */
+public class RoleInvalidationClusterTest extends AbstractInvalidationClusterTestWithTestRealm<RoleRepresentation, RoleResource> {
+
+    @Override
+    protected RoleRepresentation createTestEntityRepresentation() {
+        RoleRepresentation role = new RoleRepresentation();
+        role.setName("role_" + RandomStringUtils.randomAlphabetic(5));
+        role.setComposite(false);
+        role.setDescription("description of "+role.getName());
+        return role;
+    }
+
+    protected RolesResource roles(ContainerInfo node) {
+        return getAdminClientFor(node).realm(testRealmName).roles();
+    }
+
+    @Override
+    protected RoleResource entityResource(RoleRepresentation role, ContainerInfo node) {
+        return entityResource(role.getName(), node);
+    }
+
+    @Override
+    protected RoleResource entityResource(String name, ContainerInfo node) {
+        return roles(node).get(name);
+    }
+
+    @Override
+    protected RoleRepresentation createEntity(RoleRepresentation role, ContainerInfo node) {
+        roles(node).create(role);
+        return readEntity(role, node);
+    }
+
+    @Override
+    protected RoleRepresentation readEntity(RoleRepresentation role, ContainerInfo node) {
+        RoleRepresentation u = null;
+        try {
+            u = entityResource(role, node).toRepresentation();
+        } catch (NotFoundException nfe) {
+            // expected when role doesn't exist
+        }
+        return u;
+    }
+
+    @Override
+    protected RoleRepresentation updateEntity(RoleRepresentation role, ContainerInfo node) {
+        return updateEntity(role.getName(), role, node);
+    }
+
+    private RoleRepresentation updateEntity(String roleName, RoleRepresentation role, ContainerInfo node) {
+        entityResource(roleName, node).update(role);
+        return readEntity(role, node);
+    }
+
+    @Override
+    protected void deleteEntity(RoleRepresentation role, ContainerInfo node) {
+        entityResource(role, node).remove();
+        assertNull(readEntity(role, node));
+    }
+
+    @Override
+    protected RoleRepresentation testEntityUpdates(RoleRepresentation role, boolean backendFailover) {
+
+        // description
+        role.setDescription(role.getDescription()+"_- updated");
+        role = updateEntityOnCurrentFailNode(role, "description");
+        verifyEntityUpdateDuringFailover(role, backendFailover);
+        
+        // TODO composites
+
+        return role;
+    }
+
+}

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/SessionFailoverClusterTest.java

@@ -17,7 +17,7 @@ import org.openqa.selenium.Cookie;
  *
  * @author tkyjovsk
  */
-public class SessionFailoverClusterTest extends AbstractTwoNodeClusterTest {
+public class SessionFailoverClusterTest extends AbstractClusterTest {
 
     public static final String KEYCLOAK_SESSION_COOKIE = "KEYCLOAK_SESSION";
     public static final String KEYCLOAK_IDENTITY_COOKIE = "KEYCLOAK_IDENTITY";
@@ -30,7 +30,7 @@ public class SessionFailoverClusterTest extends AbstractTwoNodeClusterTest {
     @Ignore("work in progress") // only works with owners="2" at the moment
     public void sessionFailover() {
         
-        // LOGOUT
+        // LOGIN
         accountPage.navigateTo();
         driver.navigate().refresh();
         pause(3000);
@@ -40,7 +40,7 @@ public class SessionFailoverClusterTest extends AbstractTwoNodeClusterTest {
         Cookie sessionCookie = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
         assertNotNull(sessionCookie);
 
-        killBackend1();
+        failure();
 
         // check if session survived backend failure
         
@@ -53,6 +53,7 @@ public class SessionFailoverClusterTest extends AbstractTwoNodeClusterTest {
         assertEquals(sessionCookieAfterFailover.getValue(), sessionCookie.getValue());
 
         failback();
+        iterateCurrentFailNode();
 
         // check if session survived backend failback
         driver.navigate().refresh();
@@ -71,7 +72,7 @@ public class SessionFailoverClusterTest extends AbstractTwoNodeClusterTest {
         sessionCookie = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
         assertNull(sessionCookie);
 
-        killBackend1();
+        failure();
         
         // check if session survived backend failure
         driver.navigate().refresh();

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/UserInvalidationClusterTest.java

@@ -0,0 +1,94 @@
+package org.keycloak.testsuite.cluster;
+
+import javax.ws.rs.NotFoundException;
+import javax.ws.rs.core.Response;
+import org.apache.commons.lang.RandomStringUtils;
+import static org.junit.Assert.assertNull;
+import org.keycloak.admin.client.resource.UserResource;
+import org.keycloak.admin.client.resource.UsersResource;
+import org.keycloak.representations.idm.UserRepresentation;
+import org.keycloak.testsuite.admin.ApiUtil;
+import org.keycloak.testsuite.arquillian.ContainerInfo;
+
+/**
+ *
+ * @author tkyjovsk
+ */
+public class UserInvalidationClusterTest extends AbstractInvalidationClusterTestWithTestRealm<UserRepresentation, UserResource> {
+
+    @Override
+    protected UserRepresentation createTestEntityRepresentation() {
+        String firstName = "user";
+        String lastName = RandomStringUtils.randomAlphabetic(5);
+        UserRepresentation user = new UserRepresentation();
+        user.setUsername(firstName + "_" + lastName);
+        user.setEmail(user.getUsername() + "@email.test");
+        user.setFirstName(firstName);
+        user.setLastName(lastName);
+        return user;
+    }
+
+    protected UsersResource users(ContainerInfo node) {
+        return getAdminClientFor(node).realm(testRealmName).users();
+    }
+
+    @Override
+    protected UserResource entityResource(UserRepresentation user, ContainerInfo node) {
+        return entityResource(user.getId(), node);
+    }
+
+    @Override
+    protected UserResource entityResource(String id, ContainerInfo node) {
+        return users(node).get(id);
+    }
+
+    @Override
+    protected UserRepresentation createEntity(UserRepresentation user, ContainerInfo node) {
+        Response response = users(node).create(user);
+        String id = ApiUtil.getCreatedId(response);
+        response.close();
+        user.setId(id);
+        return readEntity(user, node);
+    }
+
+    @Override
+    protected UserRepresentation readEntity(UserRepresentation user, ContainerInfo node) {
+        UserRepresentation u = null;
+        try {
+            u = entityResource(user, node).toRepresentation();
+        } catch (NotFoundException nfe) {
+            // expected when user doesn't exist
+        }
+        return u;
+    }
+
+    @Override
+    protected UserRepresentation updateEntity(UserRepresentation user, ContainerInfo node) {
+        entityResource(user, node).update(user);
+        return readEntity(user, node);
+    }
+
+    @Override
+    protected void deleteEntity(UserRepresentation user, ContainerInfo node) {
+        entityResource(user, node).remove();
+        assertNull(readEntity(user, node));
+    }
+
+    @Override
+    protected UserRepresentation testEntityUpdates(UserRepresentation user, boolean backendFailover) {
+
+        // username
+        user.setUsername(user.getUsername() + "_updated");
+        user = updateEntityOnCurrentFailNode(user, "username");
+        verifyEntityUpdateDuringFailover(user, backendFailover);
+
+        // first+lastName
+        user.setFirstName(user.getFirstName() + "_updated");
+        user.setLastName(user.getLastName() + "_updated");
+        user = updateEntityOnCurrentFailNode(user, "firstName/lastName");
+        verifyEntityUpdateDuringFailover(user, backendFailover);
+
+        return user;
+    }
+
+}

testsuite/integration-arquillian/tests/base/src/test/resources/META-INF/keycloak-server.json

@@ -42,7 +42,6 @@
     },
 
     "theme": {
-        "default": "keycloak",
         "staticMaxAge": "${keycloak.theme.staticMaxAge:2592000}",
         "cacheTemplates": "${keycloak.theme.cacheTemplates:true}",
         "cacheThemes": "${keycloak.theme.cacheThemes:true}",

testsuite/integration-arquillian/tests/base/src/test/resources/adapter-test/keycloak-saml/sp-metadata.xml

@@ -24,9 +24,9 @@
                 protocolSupportEnumeration="urn:oasis:names:tc:SAML:2.0:protocol urn:oasis:names:tc:SAML:1.1:protocol http://schemas.xmlsoap.org/ws/2003/07/secext">
             <NameIDFormat>urn:oasis:names:tc:SAML:2.0:nameid-format:transient
             </NameIDFormat>
-            <SingleLogoutService Binding="urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST" Location="http://localhost:8080/sales-metadata/"/>
+            <SingleLogoutService Binding="urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" Location="http://localhost:8080/sales-metadata/"/>
             <AssertionConsumerService
-                    Binding="urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST" Location="http://localhost:8080/sales-metadata/"
+                    Binding="urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" Location="http://localhost:8080/sales-metadata/"
                     index="1" isDefault="true" />
             <KeyDescriptor use="signing">
                 <dsig:KeyInfo xmlns:dsig="http://www.w3.org/2000/09/xmldsig#">

testsuite/integration-arquillian/tests/base/src/test/resources/adapter-test/keycloak-saml/testsaml.json

@@ -172,8 +172,8 @@
             "enabled": true,
             "protocol": "saml",
             "fullScopeAllowed": true,
-            "baseUrl": "http://localhost:8080/bad-realm-sales-post-sig/",
-            "adminUrl": "http://localhost:8080/bad-realm-sales-post-sig/",
+            "baseUrl": "http://localhost:8080/bad-realm-sales-post-sig",
+            "adminUrl": "http://localhost:8080/bad-realm-sales-post-sig",
             "redirectUris": [
                 "http://localhost:8080/bad-realm-sales-post-sig/*"
             ],
@@ -189,8 +189,8 @@
             "enabled": true,
             "protocol": "saml",
             "fullScopeAllowed": true,
-            "baseUrl": "http://localhost:8080/bad-client-sales-post-sig/",
-            "adminUrl": "http://localhost:8080/bad-client-sales-post-sig/",
+            "baseUrl": "http://localhost:8080/bad-client-sales-post-sig",
+            "adminUrl": "http://localhost:8080/bad-client-sales-post-sig",
             "redirectUris": [
                 "http://localhost:8080/bad-client-sales-post-sig/*"
             ],
@@ -229,7 +229,7 @@
             "redirectUris": [
                 "http://localhost:8080/employee-sig/*"
             ],
-            "adminUrl": "http://localhost:8080/employee-sig/",
+            "adminUrl": "http://localhost:8080/employee-sig",
             "attributes": {
                 "saml.server.signature": "true",
                 "saml.client.signature": "true",
@@ -243,11 +243,11 @@
             "enabled": true,
             "protocol": "saml",
             "fullScopeAllowed": true,
-            "baseUrl": "http://localhost:8080/employee/",
+            "baseUrl": "http://localhost:8080/employee",
             "redirectUris": [
                 "http://localhost:8080/employee/*"
             ],
-            "adminUrl": "http://localhost:8080/employee/",
+            "adminUrl": "http://localhost:8080/employee",
             "attributes": {
                 "saml.authnstatement": "true"
             },
@@ -293,11 +293,11 @@
             "enabled": true,
             "protocol": "saml",
             "fullScopeAllowed": true,
-            "baseUrl": "http://localhost:8080/employee2/",
+            "baseUrl": "http://localhost:8080/employee2",
             "redirectUris": [
                 "http://localhost:8080/employee2/*"
             ],
-            "adminUrl": "http://localhost:8080/employee2/",
+            "adminUrl": "http://localhost:8080/employee2",
             "attributes": {
                 "saml.authnstatement": "true"
             },
@@ -344,7 +344,7 @@
             "protocol": "saml",
             "fullScopeAllowed": true,
             "frontchannelLogout": true,
-            "baseUrl": "http://localhost:8080/employee-sig-front/",
+            "baseUrl": "http://localhost:8080/employee-sig-front",
             "redirectUris": [
                 "http://localhost:8080/employee-sig-front/*"
             ],

testsuite/integration-arquillian/tests/base/src/test/resources/arquillian.xml

@@ -79,6 +79,7 @@
                     -Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m
                     -Djava.net.preferIPv4Stack=true
                 </property>
+                <property name="outputToConsole">${frontend.console.output}</property>
                 <property name="managementPort">${auth.server.management.port}</property>
                 <property name="startupTimeoutInSeconds">${startup.timeout.sec}</property>
             </configuration>
@@ -98,7 +99,7 @@
                     -Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m
                     -Djava.net.preferIPv4Stack=true
                 </property>
-                <!--<property name="outputToConsole">false</property>-->
+                <property name="outputToConsole">${backends.console.output}</property>
                 <property name="managementPort">${auth.server.backend1.management.port}</property>
                 <property name="startupTimeoutInSeconds">${startup.timeout.sec}</property>
             </configuration>
@@ -118,7 +119,7 @@
                     -Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m
                     -Djava.net.preferIPv4Stack=true
                 </property>
-                <!--<property name="outputToConsole">false</property>-->
+                <property name="outputToConsole">${backends.console.output}</property>
                 <property name="managementPort">${auth.server.backend2.management.port}</property>
                 <property name="startupTimeoutInSeconds">${startup.timeout.sec}</property>
             </configuration>

testsuite/integration-arquillian/tests/other/mod_auth_mellon/src/test/java/org/keycloak/testsuite/mod_auth_mellon/ModAuthMellonTest.java

@@ -30,20 +30,20 @@ public class ModAuthMellonTest extends AbstractAuthTest {
     @Test
     public void modAuthMellonTest() throws TransformerException {
         testRealmPage.setAuthRealm("mellon-test");
-        testRealmSAMLLoginPage.setAuthRealm("mellon-test");
+        testRealmSAMLRedirectLoginPage.setAuthRealm("mellon-test");
 
         modAuthMellonUnprotectedResourcePage.navigateTo();
         assertTrue(driver.getPageSource().contains("Unprotected resource"));
 
         modAuthMellonProtectedResourcePage.navigateTo();
-        URLAssert.assertCurrentUrlStartsWith(testRealmSAMLLoginPage);
-        testRealmSAMLLoginPage.form().login(bburkeUser);
+        URLAssert.assertCurrentUrlStartsWith(testRealmSAMLRedirectLoginPage);
+        testRealmSAMLRedirectLoginPage.form().login(bburkeUser);
         assertTrue(driver.getPageSource().contains("Protected resource"));
 
         modAuthMellonProtectedResourcePage.logout();
         assertTrue(driver.getPageSource().contains("Unprotected resource"));
 
         modAuthMellonProtectedResourcePage.navigateTo();
-        URLAssert.assertCurrentUrlStartsWith(testRealmSAMLLoginPage);
+        URLAssert.assertCurrentUrlStartsWith(testRealmSAMLRedirectLoginPage);
     }
 }

testsuite/integration-arquillian/tests/pom.xml

@@ -65,6 +65,9 @@
         <arquillian-wildfly-container.version>8.2.0.Final</arquillian-wildfly-container.version>
         <version.shrinkwrap.resolvers>2.1.1</version.shrinkwrap.resolvers>
         
+        <frontend.console.output>true</frontend.console.output>
+        <backends.console.output>true</backends.console.output>
+        
     </properties>
 
     <dependencyManagement>
@@ -119,6 +122,8 @@
                             <auth.server.ssl.required>${auth.server.ssl.required}</auth.server.ssl.required>
                             <startup.timeout.sec>${startup.timeout.sec}</startup.timeout.sec>
                             <jboss.server.config.dir>${jboss.server.config.dir}</jboss.server.config.dir>
+                            <frontend.console.output>${frontend.console.output}</frontend.console.output>
+                            <backends.console.output>${backend.console.output}</backends.console.output>
                         </systemPropertyVariables>
                         <properties>
                             <property>

testsuite/integration/src/test/java/org/keycloak/testsuite/KeycloakServer.java

@@ -33,7 +33,7 @@ import org.keycloak.services.filters.KeycloakSessionServletFilter;
 import org.keycloak.services.managers.ApplianceBootstrap;
 import org.keycloak.services.managers.RealmManager;
 import org.keycloak.services.resources.KeycloakApplication;
-import org.keycloak.testsuite.util.cli.InfinispanCLI;
+import org.keycloak.testsuite.util.cli.TestsuiteCLI;
 import org.keycloak.util.JsonSerialization;
 
 import javax.servlet.DispatcherType;
@@ -206,8 +206,8 @@ public class KeycloakServer {
             }
         });
 
-        if (System.getProperties().containsKey("startInfinispanCLI")) {
-            new InfinispanCLI(keycloak).start();
+        if (System.getProperties().containsKey("startTestsuiteCLI")) {
+            new TestsuiteCLI(keycloak).start();
         }
 
         return keycloak;

testsuite/integration/src/test/java/org/keycloak/testsuite/federation/ldap/base/LDAPSyncTest.java

@@ -43,23 +43,19 @@ import org.keycloak.services.managers.UsersSyncManager;
 import org.keycloak.testsuite.federation.ldap.FederationTestUtils;
 import org.keycloak.testsuite.rule.KeycloakRule;
 import org.keycloak.testsuite.rule.LDAPRule;
-import org.keycloak.testsuite.DummyUserFederationProviderFactory;
-import org.keycloak.timer.TimerProvider;
 import org.keycloak.common.util.Time;
 
-import java.util.HashMap;
 import java.util.Map;
 
 /**
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
 @FixMethodOrder(MethodSorters.NAME_ASCENDING)
-public class SyncProvidersTest {
+public class LDAPSyncTest {
 
     private static LDAPRule ldapRule = new LDAPRule();
 
     private static UserFederationProviderModel ldapModel = null;
-    private static UserFederationProviderModel dummyModel = null;
 
     private static KeycloakRule keycloakRule = new KeycloakRule(new KeycloakRule.KeycloakSetup() {
 
@@ -84,9 +80,6 @@ public class SyncProvidersTest {
                 LDAPObject ldapUser = FederationTestUtils.addLDAPUser(ldapFedProvider, appRealm, "user" + i, "User" + i + "FN", "User" + i + "LN", "user" + i + "@email.org", null, "12" + i);
                 FederationTestUtils.updateLDAPPassword(ldapFedProvider, ldapUser, "Password1");
             }
-
-            // Add dummy provider
-            dummyModel = appRealm.addUserFederationProvider(DummyUserFederationProviderFactory.PROVIDER_NAME, new HashMap<String, String>(), 1, "test-dummy", -1, 1, 0);
         }
     });
 
@@ -365,37 +358,6 @@ public class SyncProvidersTest {
         }
     }
 
-    @Test
-    public void testPeriodicSync() {
-        KeycloakSession session = keycloakRule.startSession();
-        try {
-            KeycloakSessionFactory sessionFactory = session.getKeycloakSessionFactory();
-            DummyUserFederationProviderFactory dummyFedFactory = (DummyUserFederationProviderFactory)sessionFactory.getProviderFactory(UserFederationProvider.class, DummyUserFederationProviderFactory.PROVIDER_NAME);
-            int full = dummyFedFactory.getFullSyncCounter();
-            int changed = dummyFedFactory.getChangedSyncCounter();
-
-            // Assert that after some period was DummyUserFederationProvider triggered
-            UsersSyncManager usersSyncManager = new UsersSyncManager();
-            usersSyncManager.bootstrapPeriodic(sessionFactory, session.getProvider(TimerProvider.class));
-            sleep(1800);
-
-            // Cancel timer
-            usersSyncManager.removePeriodicSyncForProvider(session.getProvider(TimerProvider.class), dummyModel);
-
-            // Assert that DummyUserFederationProviderFactory.syncChangedUsers was invoked
-            int newChanged = dummyFedFactory.getChangedSyncCounter();
-            Assert.assertEquals(full, dummyFedFactory.getFullSyncCounter());
-            Assert.assertTrue(newChanged > changed);
-
-            // Assert that dummy provider won't be invoked anymore
-            sleep(1800);
-            Assert.assertEquals(full, dummyFedFactory.getFullSyncCounter());
-            Assert.assertEquals(newChanged, dummyFedFactory.getChangedSyncCounter());
-        } finally {
-            keycloakRule.stopSession(session, false);
-        }
-    }
-
     private void sleep(int time) {
         try {
             Thread.sleep(time);

testsuite/integration/src/test/java/org/keycloak/testsuite/federation/sync/SyncDummyUserFederationProviderFactory.java

@@ -0,0 +1,116 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.testsuite.federation.sync;
+
+import java.util.Date;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.jboss.logging.Logger;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.KeycloakSessionFactory;
+import org.keycloak.models.KeycloakSessionTask;
+import org.keycloak.models.RealmModel;
+import org.keycloak.models.UserFederationProviderModel;
+import org.keycloak.models.UserFederationSyncResult;
+import org.keycloak.models.UserModel;
+import org.keycloak.models.utils.KeycloakModelUtils;
+import org.keycloak.testsuite.DummyUserFederationProviderFactory;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class SyncDummyUserFederationProviderFactory extends DummyUserFederationProviderFactory {
+
+    // Used during SyncFederationTest
+    static volatile CountDownLatch latch1 = new CountDownLatch(1);
+    static volatile CountDownLatch latch2 = new CountDownLatch(1);
+
+    static void restartLatches() {
+        latch1 = new CountDownLatch(1);
+        latch2 = new CountDownLatch(1);
+    }
+
+
+
+    private static final Logger logger = Logger.getLogger(SyncDummyUserFederationProviderFactory.class);
+
+    public static final String SYNC_PROVIDER_ID = "sync-dummy";
+    public static final String WAIT_TIME = "wait-time"; // waitTime before transaction is commited
+
+    @Override
+    public String getId() {
+        return SYNC_PROVIDER_ID;
+    }
+
+    @Override
+    public Set<String> getConfigurationOptions() {
+        Set<String> list = super.getConfigurationOptions();
+        list.add(WAIT_TIME);
+        return list;
+    }
+
+    @Override
+    public UserFederationSyncResult syncChangedUsers(KeycloakSessionFactory sessionFactory, final String realmId, final UserFederationProviderModel model, Date lastSync) {
+
+        KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
+
+            @Override
+            public void run(KeycloakSession session) {
+                int waitTime = Integer.parseInt(model.getConfig().get(WAIT_TIME));
+
+                logger.infof("Starting sync of changed users. Wait time is: %s", waitTime);
+
+                RealmModel realm = session.realms().getRealm(realmId);
+
+                // KEYCLOAK-2412 : Just remove and add some users for testing purposes
+                for (int i = 0; i < 10; i++) {
+                    String username = "dummyuser-" + i;
+                    UserModel user = session.userStorage().getUserByUsername(username, realm);
+
+                    if (user != null) {
+                        session.userStorage().removeUser(realm, user);
+                    }
+
+                    user = session.userStorage().addUser(realm, username);
+                }
+
+                logger.infof("Finished sync of changed users. Waiting now for %d seconds", waitTime);
+
+
+                try {
+                    latch1.await(waitTime * 1000, TimeUnit.MILLISECONDS);
+                } catch (InterruptedException ie) {
+                    Thread.currentThread().interrupt();
+                    throw new RuntimeException("Interrupted!", ie);
+                }
+
+                logger.infof("Finished waiting");
+            }
+
+        });
+
+        // countDown, so the SyncFederationTest can continue
+        latch2.countDown();
+
+        return new UserFederationSyncResult();
+    }
+
+}

testsuite/integration/src/test/java/org/keycloak/testsuite/federation/sync/SyncFederationTest.java

@@ -0,0 +1,175 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.testsuite.federation.sync;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Assert;
+import org.junit.ClassRule;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runners.MethodSorters;
+import org.keycloak.common.util.Time;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.KeycloakSessionFactory;
+import org.keycloak.models.RealmModel;
+import org.keycloak.models.UserFederationProvider;
+import org.keycloak.models.UserFederationProviderModel;
+import org.keycloak.models.UserFederationSyncResult;
+import org.keycloak.services.managers.RealmManager;
+import org.keycloak.services.managers.UsersSyncManager;
+import org.keycloak.testsuite.DummyUserFederationProviderFactory;
+import org.keycloak.testsuite.rule.KeycloakRule;
+import org.keycloak.timer.TimerProvider;
+
+/**
+ * Test with Dummy providers (For LDAP see {@link org.keycloak.testsuite.federation.ldap.base.LDAPSyncTest}
+ *
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+public class SyncFederationTest {
+
+    private static UserFederationProviderModel dummyModel = null;
+
+    @ClassRule
+    public static KeycloakRule keycloakRule = new KeycloakRule(new KeycloakRule.KeycloakSetup() {
+
+        @Override
+        public void config(RealmManager manager, RealmModel adminstrationRealm, RealmModel appRealm) {
+            // Other tests may left Time offset uncleared, which could cause issues
+            Time.setOffset(0);
+        }
+    });
+
+    @Test
+    public void test01PeriodicSync() {
+
+        // Enable timer for SyncDummyUserFederationProvider
+        keycloakRule.update(new KeycloakRule.KeycloakSetup() {
+
+            @Override
+            public void config(RealmManager manager, RealmModel adminstrationRealm, RealmModel appRealm) {
+                dummyModel = appRealm.addUserFederationProvider(DummyUserFederationProviderFactory.PROVIDER_NAME, new HashMap<String, String>(), 1, "test-sync-dummy", -1, 1, 0);
+            }
+
+        });
+
+        KeycloakSession session = keycloakRule.startSession();
+        try {
+            KeycloakSessionFactory sessionFactory = session.getKeycloakSessionFactory();
+            DummyUserFederationProviderFactory dummyFedFactory = (DummyUserFederationProviderFactory)sessionFactory.getProviderFactory(UserFederationProvider.class, DummyUserFederationProviderFactory.PROVIDER_NAME);
+            int full = dummyFedFactory.getFullSyncCounter();
+            int changed = dummyFedFactory.getChangedSyncCounter();
+
+            // Assert that after some period was DummyUserFederationProvider triggered
+            UsersSyncManager usersSyncManager = new UsersSyncManager();
+            usersSyncManager.bootstrapPeriodic(sessionFactory, session.getProvider(TimerProvider.class));
+            sleep(1800);
+
+            // Cancel timer
+            RealmModel appRealm = session.realms().getRealmByName("test");
+            usersSyncManager.notifyToRefreshPeriodicSync(session, appRealm, dummyModel, true);
+
+            // Assert that DummyUserFederationProviderFactory.syncChangedUsers was invoked
+            int newChanged = dummyFedFactory.getChangedSyncCounter();
+            Assert.assertEquals(full, dummyFedFactory.getFullSyncCounter());
+            Assert.assertTrue(newChanged > changed);
+
+            // Assert that dummy provider won't be invoked anymore
+            sleep(1800);
+            Assert.assertEquals(full, dummyFedFactory.getFullSyncCounter());
+            Assert.assertEquals(newChanged, dummyFedFactory.getChangedSyncCounter());
+        } finally {
+            keycloakRule.stopSession(session, true);
+        }
+
+        // remove dummyProvider
+        keycloakRule.update(new KeycloakRule.KeycloakSetup() {
+
+            @Override
+            public void config(RealmManager manager, RealmModel adminstrationRealm, RealmModel appRealm) {
+                appRealm.removeUserFederationProvider(dummyModel);
+            }
+
+        });
+    }
+
+    @Test
+    public void test02ConcurrentSync() throws Exception {
+        SyncDummyUserFederationProviderFactory.restartLatches();
+
+        // Enable timer for SyncDummyUserFederationProvider
+        keycloakRule.update(new KeycloakRule.KeycloakSetup() {
+
+            @Override
+            public void config(RealmManager manager, RealmModel adminstrationRealm, RealmModel appRealm) {
+                Map<String, String> config = new HashMap<>();
+                config.put(SyncDummyUserFederationProviderFactory.WAIT_TIME, "2000");
+                dummyModel = appRealm.addUserFederationProvider(SyncDummyUserFederationProviderFactory.SYNC_PROVIDER_ID, config, 1, "test-sync-dummy", -1, 1, 0);
+            }
+
+        });
+
+        KeycloakSession session = keycloakRule.startSession();
+        try {
+            KeycloakSessionFactory sessionFactory = session.getKeycloakSessionFactory();
+
+            // bootstrap periodic sync
+            UsersSyncManager usersSyncManager = new UsersSyncManager();
+            usersSyncManager.bootstrapPeriodic(sessionFactory, session.getProvider(TimerProvider.class));
+
+            // Wait and then trigger sync manually. Assert it will be ignored
+            sleep(1800);
+            RealmModel realm = session.realms().getRealm("test");
+            UserFederationSyncResult syncResult = usersSyncManager.syncChangedUsers(sessionFactory, realm.getId(), dummyModel);
+            Assert.assertTrue(syncResult.isIgnored());
+
+            // Cancel timer
+            usersSyncManager.notifyToRefreshPeriodicSync(session, realm, dummyModel, true);
+
+            // Signal to factory to finish waiting
+            SyncDummyUserFederationProviderFactory.latch1.countDown();
+
+        } finally {
+            keycloakRule.stopSession(session, true);
+        }
+
+        SyncDummyUserFederationProviderFactory.latch2.await(20000, TimeUnit.MILLISECONDS);
+
+        // remove provider
+        keycloakRule.update(new KeycloakRule.KeycloakSetup() {
+
+            @Override
+            public void config(RealmManager manager, RealmModel adminstrationRealm, RealmModel appRealm) {
+                appRealm.removeUserFederationProvider(dummyModel);
+            }
+
+        });
+    }
+
+    private void sleep(int time) {
+        try {
+            Thread.sleep(time);
+        } catch (InterruptedException ie) {
+            throw new RuntimeException(ie);
+        }
+    }
+}

testsuite/integration/src/test/java/org/keycloak/testsuite/model/UserSessionInitializerTest.java

@@ -26,6 +26,7 @@ import org.junit.Assert;
 import org.junit.Before;
 import org.junit.ClassRule;
 import org.junit.Test;
+import org.keycloak.cluster.ClusterProvider;
 import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
 import org.keycloak.models.ClientModel;
 import org.keycloak.models.ClientSessionModel;
@@ -83,7 +84,7 @@ public class UserSessionInitializerTest {
 
         // Create and persist offline sessions
         int started = Time.currentTime();
-        int serverStartTime = session.sessions().getClusterStartupTime();
+        int serverStartTime = session.getProvider(ClusterProvider.class).getClusterStartupTime();
 
         for (UserSessionModel origSession : origSessions) {
             UserSessionModel userSession = session.sessions().getUserSession(realm, origSession.getId());
@@ -99,7 +100,7 @@ public class UserSessionInitializerTest {
 
         // Clear ispn cache to ensure initializerState is removed as well
         InfinispanConnectionProvider infinispan = session.getProvider(InfinispanConnectionProvider.class);
-        infinispan.getCache(InfinispanConnectionProvider.OFFLINE_SESSION_CACHE_NAME).clear();
+        infinispan.getCache(InfinispanConnectionProvider.WORK_CACHE_NAME).clear();
 
         resetSession();
 

testsuite/integration/src/test/java/org/keycloak/testsuite/util/cli/AbstractCommand.java

@@ -35,7 +35,7 @@ public abstract class AbstractCommand {
     protected List<String> args;
     protected KeycloakSessionFactory sessionFactory;
 
-    public void injectProperties(List<String> args, InfinispanCLI cli, KeycloakSessionFactory sessionFactory) {
+    public void injectProperties(List<String> args, TestsuiteCLI cli, KeycloakSessionFactory sessionFactory) {
         this.args = args;
         this.sessionFactory = sessionFactory;
     }

testsuite/integration/src/test/java/org/keycloak/testsuite/util/cli/PersistSessionsCommand.java

@@ -43,10 +43,32 @@ public class PersistSessionsCommand extends AbstractCommand {
     @Override
     public void doRunCommand(KeycloakSession sess) {
         final int count = getIntArg(0);
+        final int batchCount = getIntArg(1);
+
+        int remaining = count;
+
+        while (remaining > 0) {
+            int createInThisBatch = Math.min(batchCount, remaining);
+            createSessionsBatch(createInThisBatch);
+            remaining = remaining - createInThisBatch;
+        }
+
+        // Write some summary
+        KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
+
+            @Override
+            public void run(KeycloakSession session) {
+                UserSessionPersisterProvider persister = session.getProvider(UserSessionPersisterProvider.class);
+                log.info("Command finished. Total number of sessions in persister: " + persister.getUserSessionsCount(true));
+            }
+
+        });
+    }
+
+    private void createSessionsBatch(final int countInThisBatch) {
         final List<String> userSessionIds = new LinkedList<>();
         final List<String> clientSessionIds = new LinkedList<>();
 
-        // Create sessions in separate transaction first
         KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
 
             @Override
@@ -56,7 +78,7 @@ public class PersistSessionsCommand extends AbstractCommand {
                 ClientModel testApp = realm.getClientByClientId("security-admin-console");
                 UserSessionPersisterProvider persister = session.getProvider(UserSessionPersisterProvider.class);
 
-                for (int i = 0; i < count; i++) {
+                for (int i = 0; i < countInThisBatch; i++) {
                     UserSessionModel userSession = session.sessions().createUserSession(realm, john, "john-doh@localhost", "127.0.0.2", "form", true, null, null);
                     ClientSessionModel clientSession = session.sessions().createClientSession(realm, testApp);
                     clientSession.setUserSession(userSession);
@@ -69,9 +91,10 @@ public class PersistSessionsCommand extends AbstractCommand {
 
         });
 
-        log.info("Sessions created in infinispan storage");
+        log.infof("%d sessions created in infinispan storage", countInThisBatch);
 
         // Persist them now
+
         KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
 
             @Override
@@ -84,35 +107,17 @@ public class PersistSessionsCommand extends AbstractCommand {
                     counter++;
                     UserSessionModel userSession = session.sessions().getUserSession(realm, userSessionId);
                     persister.createUserSession(userSession, true);
-                    if (counter%1000 == 0) {
-                        log.infof("%d user sessions persisted. Continue", counter);
-                    }
                 }
 
-                log.infof("All %d user sessions persisted", counter);
+                log.infof("%d user sessions persisted. Continue", counter);
 
                 counter = 0;
                 for (String clientSessionId : clientSessionIds) {
                     counter++;
                     ClientSessionModel clientSession = session.sessions().getClientSession(realm, clientSessionId);
                     persister.createClientSession(clientSession, true);
-                    if (counter%1000 == 0) {
-                        log.infof("%d client sessions persisted. Continue", counter);
-                    }
                 }
-
-                log.infof("All %d client sessions persisted", counter);
-            }
-
-        });
-
-        // Persist them now
-        KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
-
-            @Override
-            public void run(KeycloakSession session) {
-                UserSessionPersisterProvider persister = session.getProvider(UserSessionPersisterProvider.class);
-                log.info("Total number of sessions in persister: " + persister.getUserSessionsCount(true));
+                log.infof("%d client sessions persisted. Continue", counter);
             }
 
         });
@@ -120,6 +125,6 @@ public class PersistSessionsCommand extends AbstractCommand {
 
     @Override
     public String printUsage() {
-        return super.printUsage() + " <sessions-count>";
+        return super.printUsage() + " <sessions-count> <sessions-count-per-each-transaction>";
     }
 }

testsuite/integration/src/test/java/org/keycloak/testsuite/util/cli/SyncDummyFederationProviderCommand.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.testsuite.util.cli;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.RealmModel;
+import org.keycloak.models.UserFederationProviderModel;
+import org.keycloak.models.utils.KeycloakModelUtils;
+import org.keycloak.services.managers.UsersSyncManager;
+import org.keycloak.testsuite.federation.sync.SyncDummyUserFederationProviderFactory;
+import org.keycloak.timer.TimerProvider;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class SyncDummyFederationProviderCommand extends AbstractCommand {
+
+    @Override
+    protected void doRunCommand(KeycloakSession session) {
+        int waitTime = getIntArg(0);
+        int changedSyncPeriod = getIntArg(1);
+
+        RealmModel realm = session.realms().getRealmByName("master");
+        UserFederationProviderModel fedProviderModel = KeycloakModelUtils.findUserFederationProviderByDisplayName("cluster-dummy", realm);
+        if (fedProviderModel == null) {
+            Map<String, String> cfg = new HashMap<>();
+            updateConfig(cfg, waitTime);
+            fedProviderModel = realm.addUserFederationProvider(SyncDummyUserFederationProviderFactory.SYNC_PROVIDER_ID, cfg, 1, "cluster-dummy", -1, changedSyncPeriod, -1);
+        } else {
+            Map<String, String> cfg = fedProviderModel.getConfig();
+            updateConfig(cfg, waitTime);
+            fedProviderModel.setChangedSyncPeriod(changedSyncPeriod);
+            realm.updateUserFederationProvider(fedProviderModel);
+        }
+
+        new UsersSyncManager().notifyToRefreshPeriodicSync(session, realm, fedProviderModel, false);
+
+        log.infof("User federation provider created and sync was started", waitTime);
+    }
+
+    private void updateConfig(Map<String, String> cfg, int waitTime) {
+        cfg.put(SyncDummyUserFederationProviderFactory.WAIT_TIME, String.valueOf(waitTime));
+    }
+
+
+    @Override
+    public String getName() {
+        return "startSyncDummy";
+    }
+
+    @Override
+    public String printUsage() {
+        return super.printUsage() + " <wait-time-before-sync-commit-in-seconds> <changed-sync-period-in-seconds>";
+    }
+}

testsuite/integration/src/test/java/org/keycloak/testsuite/util/cli/TestsuiteCLI.java

@@ -32,27 +32,13 @@ import org.keycloak.models.KeycloakSessionFactory;
 import org.keycloak.testsuite.KeycloakServer;
 
 /**
- * HOWTO USE THIS:
- *
- * 1) Run KeycloakServer with system properties (assuming mongo up and running on localhost):
- *      -Dkeycloak.realm.provider=mongo -Dkeycloak.user.provider=mongo -Dkeycloak.userSessionPersister.provider=mongo -Dkeycloak.connectionsMongo.db=keycloak -Dkeycloak.connectionsInfinispan.clustered=true -Dresources -DstartInfinispanCLI
- *
- * 2) Write command on STDIN to persist 50000 userSessions to mongo: persistSessions 50000
- *
- * 3) Run command "clear" to ensure infinispan cache is cleared. Doublecheck with command "size" is 0
- *
- * 4) Write command to load sessions from persistent storage - 100 sessions per worker transaction: loadPersistentSessions 100
- *
- * See the progress in log. Finally run command "size" to ensure size is 100001 (50000 userSessions + 50000 clientSessions + 1 initializationState item)
- *
- * 5) Alternative to step 3+4 - Kill the server after step 2 and start two KeycloakServer in parallel on ports 8081 and 8082 . See the progress in logs of loading persistent sessions to infinispan.
- * Kill the coordinator (usually 8081 node) during startup and see the node 8082 became coordinator and took ownership of loading persistent sessions. After node 8082 fully started, the size of infinispan is again 100001
+ * See Testsuite.md (section how to create many users and offline sessions)
  *
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
-public class InfinispanCLI {
+public class TestsuiteCLI {
 
-    private static final Logger log = Logger.getLogger(InfinispanCLI.class);
+    private static final Logger log = Logger.getLogger(TestsuiteCLI.class);
 
     private static final Class<?>[] BUILTIN_COMMANDS = {
             ExitCommand.class,
@@ -69,13 +55,14 @@ public class InfinispanCLI {
             UserCommands.Create.class,
             UserCommands.Remove.class,
             UserCommands.Count.class,
-            UserCommands.GetUser.class
+            UserCommands.GetUser.class,
+            SyncDummyFederationProviderCommand.class
     };
 
     private final KeycloakSessionFactory sessionFactory;
     private final Map<String, Class<? extends AbstractCommand>> commands = new LinkedHashMap<>();
 
-    public InfinispanCLI(KeycloakServer server) {
+    public TestsuiteCLI(KeycloakServer server) {
         this.sessionFactory = server.getSessionFactory();
 
         // register builtin commands
@@ -96,7 +83,7 @@ public class InfinispanCLI {
 
     // WARNING: Stdin blocking operation
     public void start() throws IOException {
-        log.info("Starting infinispan CLI. Exit with 'exit' . Available commands with 'help' ");
+        log.info("Starting testsuite CLI. Exit with 'exit' . Available commands with 'help' ");
 
         BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));
         String line;
@@ -131,7 +118,7 @@ public class InfinispanCLI {
                 System.out.print("$ ");
             }
         } finally {
-            log.info("Exit infinispan CLI");
+            log.info("Exit testsuite CLI");
             reader.close();
         }
     }
@@ -164,7 +151,7 @@ public class InfinispanCLI {
         private List<String> commandNames = new ArrayList<>();
 
         @Override
-        public void injectProperties(List<String> args, InfinispanCLI cli, KeycloakSessionFactory sessionFactory) {
+        public void injectProperties(List<String> args, TestsuiteCLI cli, KeycloakSessionFactory sessionFactory) {
             for (String commandName : cli.commands.keySet()) {
                 commandNames.add(commandName);
             }

testsuite/integration/src/test/resources/META-INF/keycloak-server.json

@@ -37,7 +37,6 @@
     },
 
     "theme": {
-        "default": "keycloak",
         "staticMaxAge": "${keycloak.theme.staticMaxAge:2592000}",
         "cacheTemplates": "${keycloak.theme.cacheTemplates:true}",
         "cacheThemes": "${keycloak.theme.cacheThemes:true}",

testsuite/integration/src/test/resources/META-INF/services/org.keycloak.models.UserFederationProviderFactory

@@ -15,4 +15,5 @@
 # limitations under the License.
 #
 
-org.keycloak.testsuite.DummyUserFederationProviderFactory
+org.keycloak.testsuite.DummyUserFederationProviderFactory
+org.keycloak.testsuite.federation.sync.SyncDummyUserFederationProviderFactory

wildfly/server-subsystem/src/main/java/org/keycloak/subsystem/server/extension/KeycloakServerDeploymentProcessor.java

@@ -62,6 +62,7 @@ public class KeycloakServerDeploymentProcessor implements DeploymentUnitProcesso
             st.addDependency(cacheContainerService.append("sessions"));
             st.addDependency(cacheContainerService.append("offlineSessions"));
             st.addDependency(cacheContainerService.append("loginFailures"));
+            st.addDependency(cacheContainerService.append("work"));
             st.addDependency(cacheContainerService.append("realmVersions"));
         }
     }

wildfly/server-subsystem/src/main/resources/subsystem-templates/keycloak-infinispan.xml

@@ -30,6 +30,7 @@
                 <local-cache name="sessions"/>
                 <local-cache name="offlineSessions"/>
                 <local-cache name="loginFailures"/>
+                <local-cache name="work"/>
                 <local-cache name="realmVersions">
                     <transaction mode="BATCH" locking="PESSIMISTIC"/>
                 </local-cache>
@@ -90,6 +91,7 @@
                 <distributed-cache name="sessions" mode="SYNC" owners="1"/>
                 <distributed-cache name="offlineSessions" mode="SYNC" owners="1"/>
                 <distributed-cache name="loginFailures" mode="SYNC" owners="1"/>
+                <replicated-cache name="work" mode="SYNC" />
                 <local-cache name="realmVersions">
                     <transaction mode="BATCH" locking="PESSIMISTIC"/>
                 </local-cache>