Merge pull request #3511 from mposolda/ispn-invalidations-rebase

KEYCLOAK-3857 Clustered invalidation cache fixes and refactoring. Sup…

distribution/feature-packs/server-feature-pack/src/main/resources/modules/system/layers/keycloak/org/keycloak/keycloak-model-infinispan/main/module.xml

@@ -30,6 +30,9 @@
         <module name="org.keycloak.keycloak-server-spi"/>
         <module name="org.keycloak.keycloak-server-spi-private"/>
         <module name="org.infinispan"/>
+        <module name="org.infinispan.commons"/>
+        <module name="org.infinispan.cachestore.remote"/>
+        <module name="org.infinispan.client.hotrod"/>
         <module name="org.jboss.logging"/>
         <module name="javax.api"/>
     </dependencies>

distribution/server-overlay/src/main/cli/keycloak-install-ha.cli

@@ -2,9 +2,9 @@ embed-server --server-config=standalone-ha.xml
 /subsystem=datasources/data-source=KeycloakDS/:add(connection-url="jdbc:h2:${jboss.server.data.dir}/keycloak;AUTO_SERVER=TRUE",jta=false,driver-name=h2,jndi-name=java:jboss/datasources/KeycloakDS,password=sa,user-name=sa,use-java-context=true)
 /subsystem=infinispan/cache-container=keycloak:add(jndi-name="infinispan/Keycloak")
 /subsystem=infinispan/cache-container=keycloak/transport=TRANSPORT:add(lock-timeout=60000)
-/subsystem=infinispan/cache-container=keycloak/invalidation-cache=realms:add(mode="SYNC")
+/subsystem=infinispan/cache-container=keycloak/local-cache=realms:add()
-/subsystem=infinispan/cache-container=keycloak/invalidation-cache=users:add(mode="SYNC")
+/subsystem=infinispan/cache-container=keycloak/local-cache=users:add()
-/subsystem=infinispan/cache-container=keycloak/invalidation-cache=users/eviction=EVICTION:add(max-entries=10000,strategy=LRU)
+/subsystem=infinispan/cache-container=keycloak/local-cache=users/eviction=EVICTION:add(max-entries=10000,strategy=LRU)
 /subsystem=infinispan/cache-container=keycloak/distributed-cache=sessions:add(mode="SYNC",owners="1")
 /subsystem=infinispan/cache-container=keycloak/distributed-cache=offlineSessions:add(mode="SYNC",owners="1")
 /subsystem=infinispan/cache-container=keycloak/distributed-cache=loginFailures:add(mode="SYNC",owners="1")

misc/CrossDataCenter.md

@@ -0,0 +1,116 @@
+Test Cross-Data-Center scenario (test with external JDG server)
+===============================================================
+
+These are temporary notes. This docs should be removed once we have cross-DC support finished and properly documented. 
+
+What is working right now is:
+- Propagating of invalidation messages for "realms" and "users" caches
+- All the other things provided by ClusterProvider, which is:
+-- ClusterStartupTime (used for offlineSessions and revokeRefreshToken) is shared for all clusters in all datacenters
+-- Periodic userStorage synchronization is always executed just on one node at a time. It won't be never executed concurrently on more nodes (Assuming "nodes" refer to all servers in all clusters in all datacenters)
+
+What doesn't work right now:
+- UserSessionProvider and offline sessions
+  
+
+Basic setup
+===========
+
+This is setup with 2 keycloak nodes, which are NOT in cluster. They just share the same database and they will be configured with "work" infinispan cache with remoteStore, which will point
+to external JDG server.
+ 
+JDG Server setup
+----------------
+- Download JDG 7.0 server and unzip to some folder
+
+- Add this into JDG_HOME/standalone/configuration/standalone.xml under cache-container named "local" :
+
+```
+<local-cache name="work" start="EAGER" batching="false" />
+```
+
+- Start server:
+```
+cd JDG_HOME/bin
+./standalone.sh -Djboss.socket.binding.port-offset=100
+```
+
+Keycloak servers setup
+----------------------
+You need to setup 2 Keycloak nodes in this way. 
+
+For now, it's recommended to test Keycloak overlay on EAP7 because of infinispan bug, which is fixed in EAP 7.0 (infinispan 8.1.2), but not 
+yet on Wildfly 10 (infinispan 8.1.0). See below for details.
+
+1) Configure shared database in KEYCLOAK_HOME/standalone/configuration/standalone.xml . For example MySQL
+
+2) Add `module` attribute to the infinispan keycloak container:
+  
+```  
+<cache-container name="keycloak" jndi-name="infinispan/Keycloak" module="org.keycloak.keycloak-model-infinispan">
+```
+  
+3) Configure `work` cache to use remoteStore. You should use this:  
+
+```
+<local-cache name="work">
+    <remote-store passivation="false" fetch-state="false" purge="false" preload="false" shared="true" cache="work" remote-servers="remote-cache">    
+        <property name="rawValues">true</property>
+        <property name="marshaller">org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory</property>
+    </remote-store>
+</local-cache>  
+```
+
+4) Configure connection to the external JDG server. Because we used port offset 100 for JDG (see above), the HotRod endpoint is running on 11322 . 
+So add the config like this to the bottom of standalone.xml under `socket-binding-group` element:
+
+```
+<outbound-socket-binding name="remote-cache">
+    <remote-destination host="localhost" port="11322"/>
+</outbound-socket-binding>
+```
+
+5) Optional: Configure logging in standalone.xml to see what invalidation events were send:
+````
+<logger category="org.keycloak.cluster.infinispan">
+    <level name="TRACE"/>
+</logger>
+<logger category="org.keycloak.models.cache.infinispan">
+    <level name="DEBUG"/>
+</logger>
+````
+           
+6)  Setup Keycloak node2 . Just copy Keycloak to another location on your laptop and repeat steps 1-5 above for second server too.
+          
+7) Run server 1 with parameters like (assuming you have virtual hosts "node1" and "node2" defined in your `/etc/hosts` ):
+```           
+./standalone.sh -Djboss.node.name=node1 -b node1 -bmanagement node1
+```
+
+and server2 with:
+```
+./standalone.sh -Djboss.node.name=node2 -b node2 -bmanagement node2
+```
+
+8) Note something like this in both `KEYCLOAK_HOME/standalone/log/server.log` on both nodes. Note that cluster Startup Time will be same time on both nodes:
+```
+2016-11-16 22:12:52,080 DEBUG [org.keycloak.cluster.infinispan.InfinispanClusterProviderFactory] (ServerService Thread Pool -- 62) My address: node1-1953169551
+2016-11-16 22:12:52,081 DEBUG [org.keycloak.cluster.infinispan.CrossDCAwareCacheFactory] (ServerService Thread Pool -- 62) RemoteStore is available. Cross-DC scenario will be used
+2016-11-16 22:12:52,119 DEBUG [org.keycloak.cluster.infinispan.InfinispanClusterProviderFactory] (ServerService Thread Pool -- 62) Loaded cluster startup time: Wed Nov 16 22:09:48 CET 2016
+2016-11-16 22:12:52,128 DEBUG [org.keycloak.cluster.infinispan.InfinispanNotificationsManager] (ServerService Thread Pool -- 62) Added listener for HotRod remoteStore cache: work
+```
+
+9) Login to node1. Then change any realm on node2. You will see in the node2 server.log that RealmUpdatedEvent was sent and on node1 that this event was received. 
+
+This is done even if node1 and node2 are NOT in cluster as it's the external JDG used for communication between 2 keycloak servers and sending/receiving cache invalidation events. But note that userSession
+doesn't yet work (eg. if you login to node1, you won't see the userSession on node2).
+
+
+WARNING: Previous steps works on Keycloak server overlay deployed on EAP 7.0 . With deploy on Wildfly 10.0.0.Final, you will see exception 
+at startup caused by the bug https://issues.jboss.org/browse/ISPN-6203 .
+
+There is a workaround to add this line into KEYCLOAK_HOME/modules/system/layers/base/org/wildfly/clustering/service/main/module.xml :
+
+```
+<module name="org.infinispan.client.hotrod"/>
+```

model/infinispan/pom.xml

@@ -48,6 +48,10 @@
             <groupId>org.infinispan</groupId>
             <artifactId>infinispan-core</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.infinispan</groupId>
+            <artifactId>infinispan-cachestore-remote</artifactId>
+        </dependency>
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>

model/infinispan/src/main/java/org/keycloak/cluster/infinispan/CrossDCAwareCacheFactory.java

@@ -0,0 +1,93 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.cluster.infinispan;
+
+import java.io.Serializable;
+import java.util.Set;
+
+import org.infinispan.Cache;
+import org.infinispan.client.hotrod.Flag;
+import org.infinispan.client.hotrod.RemoteCache;
+import org.infinispan.commons.api.BasicCache;
+import org.infinispan.persistence.remote.RemoteStore;
+import org.jboss.logging.Logger;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+abstract class CrossDCAwareCacheFactory {
+
+    protected static final Logger logger = Logger.getLogger(CrossDCAwareCacheFactory.class);
+
+
+    abstract BasicCache<String, Serializable> getCache();
+
+
+    static CrossDCAwareCacheFactory getFactory(Cache<String, Serializable> workCache, Set<RemoteStore> remoteStores) {
+        if (remoteStores.isEmpty()) {
+            logger.debugf("No configured remoteStore available. Cross-DC scenario is not used");
+            return new InfinispanCacheWrapperFactory(workCache);
+        } else {
+            logger.debugf("RemoteStore is available. Cross-DC scenario will be used");
+
+            if (remoteStores.size() > 1) {
+                logger.warnf("More remoteStores configured for work cache. Will use just the first one");
+            }
+
+            // For cross-DC scenario, we need to return underlying remoteCache for atomic operations to work properly
+            RemoteStore remoteStore = remoteStores.iterator().next();
+            RemoteCache remoteCache = remoteStore.getRemoteCache();
+            return new RemoteCacheWrapperFactory(remoteCache);
+        }
+    }
+
+
+    // We don't have external JDG configured. No cross-DC.
+    private static class InfinispanCacheWrapperFactory extends CrossDCAwareCacheFactory {
+
+        private final Cache<String, Serializable> workCache;
+
+        InfinispanCacheWrapperFactory(Cache<String, Serializable> workCache) {
+            this.workCache = workCache;
+        }
+
+        @Override
+        BasicCache<String, Serializable> getCache() {
+            return workCache;
+        }
+
+    }
+
+
+    // We have external JDG configured. Cross-DC should be enabled
+    private static class RemoteCacheWrapperFactory extends CrossDCAwareCacheFactory {
+
+        private final RemoteCache<String, Serializable> remoteCache;
+
+        RemoteCacheWrapperFactory(RemoteCache<String, Serializable> remoteCache) {
+            this.remoteCache = remoteCache;
+        }
+
+        @Override
+        BasicCache<String, Serializable> getCache() {
+            // Flags are per-invocation!
+            return remoteCache.withFlags(Flag.FORCE_RETURN_VALUE);
+        }
+
+    }
+}

model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanClusterProvider.java

@@ -17,20 +17,15 @@
 
 package org.keycloak.cluster.infinispan;
 
-import org.infinispan.Cache;
-import org.infinispan.context.Flag;
-import org.infinispan.lifecycle.ComponentStatus;
-import org.infinispan.remoting.transport.Transport;
 import org.jboss.logging.Logger;
 import org.keycloak.cluster.ClusterEvent;
 import org.keycloak.cluster.ClusterListener;
 import org.keycloak.cluster.ClusterProvider;
 import org.keycloak.cluster.ExecutionResult;
 import org.keycloak.common.util.Time;
-import org.keycloak.models.KeycloakSession;
 
-import java.io.Serializable;
 import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
 
 /**
  *
@@ -43,34 +38,22 @@ public class InfinispanClusterProvider implements ClusterProvider {
     public static final String CLUSTER_STARTUP_TIME_KEY = "cluster-start-time";
     private static final String TASK_KEY_PREFIX = "task::";
 
-    private final InfinispanClusterProviderFactory factory;
+    private final int clusterStartupTime;
-    private final KeycloakSession session;
+    private final String myAddress;
-    private final Cache<String, Serializable> cache;
+    private final CrossDCAwareCacheFactory crossDCAwareCacheFactory;
+    private final InfinispanNotificationsManager notificationsManager; // Just to extract notifications related stuff to separate class
 
-    public InfinispanClusterProvider(InfinispanClusterProviderFactory factory, KeycloakSession session, Cache<String, Serializable> cache) {
+    public InfinispanClusterProvider(int clusterStartupTime, String myAddress, CrossDCAwareCacheFactory crossDCAwareCacheFactory, InfinispanNotificationsManager notificationsManager) {
-        this.factory = factory;
+        this.myAddress = myAddress;
-        this.session = session;
+        this.clusterStartupTime = clusterStartupTime;
-        this.cache = cache;
+        this.crossDCAwareCacheFactory = crossDCAwareCacheFactory;
+        this.notificationsManager = notificationsManager;
     }
 
 
     @Override
     public int getClusterStartupTime() {
-        Integer existingClusterStartTime = (Integer) cache.get(InfinispanClusterProvider.CLUSTER_STARTUP_TIME_KEY);
+        return clusterStartupTime;
-        if (existingClusterStartTime != null) {
-            return existingClusterStartTime;
-        } else {
-            // clusterStartTime not yet initialized. Let's try to put our startupTime
-            int serverStartTime = (int) (session.getKeycloakSessionFactory().getServerStartupTimestamp() / 1000);
-
-            existingClusterStartTime = (Integer) cache.putIfAbsent(InfinispanClusterProvider.CLUSTER_STARTUP_TIME_KEY, serverStartTime);
-            if (existingClusterStartTime == null) {
-                logger.debugf("Initialized cluster startup time to %s", Time.toDate(serverStartTime).toString());
-                return serverStartTime;
-            } else {
-                return existingClusterStartTime;
-            }
-        }
     }
 
 
@@ -104,56 +87,33 @@ public class InfinispanClusterProvider implements ClusterProvider {
 
     @Override
     public void registerListener(String taskKey, ClusterListener task) {
-        factory.registerListener(taskKey, task);
+        this.notificationsManager.registerListener(taskKey, task);
     }
 
 
     @Override
-    public void notify(String taskKey, ClusterEvent event) {
+    public void notify(String taskKey, ClusterEvent event, boolean ignoreSender) {
-        // Put the value to the cache to notify listeners on all the nodes
+        this.notificationsManager.notify(taskKey, event, ignoreSender);
-        cache.put(taskKey, event);
     }
 
 
-    private String getCurrentNode(Cache<String, Serializable> cache) {
+    private LockEntry createLockEntry() {
-        Transport transport = cache.getCacheManager().getTransport();
-        return transport==null ? "local" : transport.getAddress().toString();
-    }
-
-
-    private LockEntry createLockEntry(Cache<String, Serializable> cache) {
         LockEntry lock = new LockEntry();
-        lock.setNode(getCurrentNode(cache));
+        lock.setNode(myAddress);
         lock.setTimestamp(Time.currentTime());
         return lock;
     }
 
 
     private boolean tryLock(String cacheKey, int taskTimeoutInSeconds) {
-        LockEntry myLock = createLockEntry(cache);
+        LockEntry myLock = createLockEntry();
 
-        LockEntry existingLock = (LockEntry) cache.putIfAbsent(cacheKey, myLock);
+        LockEntry existingLock = (LockEntry) crossDCAwareCacheFactory.getCache().putIfAbsent(cacheKey, myLock, taskTimeoutInSeconds, TimeUnit.SECONDS);
         if (existingLock != null) {
-            // Task likely already in progress. Check if timestamp is not outdated
+            if (logger.isTraceEnabled()) {
-            int thatTime = existingLock.getTimestamp();
+                logger.tracef("Task %s in progress already by node %s. Ignoring task.", cacheKey, existingLock.getNode());
-            int currentTime = Time.currentTime();
-            if (thatTime + taskTimeoutInSeconds < currentTime) {
-                if (logger.isTraceEnabled()) {
-                    logger.tracef("Task %s outdated when in progress by node %s. Will try to replace task with our node %s", cacheKey, existingLock.getNode(), myLock.getNode());
-                }
-                boolean replaced = cache.replace(cacheKey, existingLock, myLock);
-                if (!replaced) {
-                    if (logger.isTraceEnabled()) {
-                        logger.tracef("Failed to replace the task %s. Other thread replaced in the meantime. Ignoring task.", cacheKey);
-                    }
-                }
-                return replaced;
-            } else {
-                if (logger.isTraceEnabled()) {
-                    logger.tracef("Task %s in progress already by node %s. Ignoring task.", cacheKey, existingLock.getNode());
-                }
-                return false;
             }
+            return false;
         } else {
             if (logger.isTraceEnabled()) {
                 logger.tracef("Successfully acquired lock for task %s. Our node is %s", cacheKey, myLock.getNode());
@@ -168,20 +128,12 @@ public class InfinispanClusterProvider implements ClusterProvider {
         int retry = 3;
         while (true) {
             try {
-                cache.getAdvancedCache()
+                crossDCAwareCacheFactory.getCache().remove(cacheKey);
-                        .withFlags(Flag.IGNORE_RETURN_VALUES, Flag.FORCE_SYNCHRONOUS)
-                        .remove(cacheKey);
                 if (logger.isTraceEnabled()) {
                     logger.tracef("Task %s removed from the cache", cacheKey);
                 }
                 return;
             } catch (RuntimeException e) {
-                ComponentStatus status = cache.getStatus();
-                if (status.isStopping() || status.isTerminated()) {
-                    logger.warnf("Failed to remove task %s from the cache. Cache is already terminating", cacheKey);
-                    logger.debug(e.getMessage(), e);
-                    return;
-                }
                 retry--;
                 if (retry == 0) {
                     throw e;

model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanClusterProviderFactory.java

@@ -20,27 +20,24 @@ package org.keycloak.cluster.infinispan;
 import org.infinispan.Cache;
 import org.infinispan.manager.EmbeddedCacheManager;
 import org.infinispan.notifications.Listener;
-import org.infinispan.notifications.cachelistener.annotation.CacheEntryCreated;
-import org.infinispan.notifications.cachelistener.annotation.CacheEntryModified;
-import org.infinispan.notifications.cachelistener.event.CacheEntryCreatedEvent;
-import org.infinispan.notifications.cachelistener.event.CacheEntryModifiedEvent;
 import org.infinispan.notifications.cachemanagerlistener.annotation.ViewChanged;
 import org.infinispan.notifications.cachemanagerlistener.event.ViewChangedEvent;
+import org.infinispan.persistence.manager.PersistenceManager;
+import org.infinispan.persistence.remote.RemoteStore;
 import org.infinispan.remoting.transport.Address;
 import org.infinispan.remoting.transport.Transport;
 import org.jboss.logging.Logger;
 import org.keycloak.Config;
-import org.keycloak.cluster.ClusterEvent;
-import org.keycloak.cluster.ClusterListener;
 import org.keycloak.cluster.ClusterProvider;
 import org.keycloak.cluster.ClusterProviderFactory;
+import org.keycloak.common.util.HostUtils;
+import org.keycloak.common.util.Time;
 import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
 
 import java.io.Serializable;
 import java.util.Collection;
-import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Set;
@@ -49,6 +46,8 @@ import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
 /**
+ * This impl is aware of Cross-Data-Center scenario too
+ *
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
 public class InfinispanClusterProviderFactory implements ClusterProviderFactory {
@@ -57,28 +56,82 @@ public class InfinispanClusterProviderFactory implements ClusterProviderFactory
 
     protected static final Logger logger = Logger.getLogger(InfinispanClusterProviderFactory.class);
 
+    // Infinispan cache
     private volatile Cache<String, Serializable> workCache;
 
-    private Map<String, ClusterListener> listeners = new HashMap<>();
+    // Ensure that atomic operations (like putIfAbsent) must work correctly in any of: non-clustered, clustered or cross-Data-Center (cross-DC) setups
+    private CrossDCAwareCacheFactory crossDCAwareCacheFactory;
+
+    private String myAddress;
+
+    private int clusterStartupTime;
+
+    // Just to extract notifications related stuff to separate class
+    private InfinispanNotificationsManager notificationsManager;
 
     @Override
     public ClusterProvider create(KeycloakSession session) {
         lazyInit(session);
-        return new InfinispanClusterProvider(this, session, workCache);
+        return new InfinispanClusterProvider(clusterStartupTime, myAddress, crossDCAwareCacheFactory, notificationsManager);
     }
 
     private void lazyInit(KeycloakSession session) {
         if (workCache == null) {
             synchronized (this) {
                 if (workCache == null) {
-                    workCache = session.getProvider(InfinispanConnectionProvider.class).getCache(InfinispanConnectionProvider.WORK_CACHE_NAME);
+                    InfinispanConnectionProvider ispnConnections = session.getProvider(InfinispanConnectionProvider.class);
+                    workCache = ispnConnections.getCache(InfinispanConnectionProvider.WORK_CACHE_NAME);
+
                     workCache.getCacheManager().addListener(new ViewChangeListener());
-                    workCache.addListener(new CacheEntryListener());
+                    initMyAddress();
+
+                    Set<RemoteStore> remoteStores = getRemoteStores();
+                    crossDCAwareCacheFactory = CrossDCAwareCacheFactory.getFactory(workCache, remoteStores);
+
+                    clusterStartupTime = initClusterStartupTime(session);
+
+                    notificationsManager = InfinispanNotificationsManager.create(workCache, myAddress, remoteStores);
                 }
             }
         }
     }
 
+
+    // See if we have RemoteStore (external JDG) configured for cross-Data-Center scenario
+    private Set<RemoteStore> getRemoteStores() {
+        return workCache.getAdvancedCache().getComponentRegistry().getComponent(PersistenceManager.class).getStores(RemoteStore.class);
+    }
+
+
+    protected void initMyAddress() {
+        Transport transport = workCache.getCacheManager().getTransport();
+        this.myAddress = transport == null ? HostUtils.getHostName() + "-" + workCache.hashCode() : transport.getAddress().toString();
+        logger.debugf("My address: %s", this.myAddress);
+    }
+
+
+    protected int initClusterStartupTime(KeycloakSession session) {
+        Integer existingClusterStartTime = (Integer) crossDCAwareCacheFactory.getCache().get(InfinispanClusterProvider.CLUSTER_STARTUP_TIME_KEY);
+        if (existingClusterStartTime != null) {
+            logger.debugf("Loaded cluster startup time: %s", Time.toDate(existingClusterStartTime).toString());
+            return existingClusterStartTime;
+        } else {
+            // clusterStartTime not yet initialized. Let's try to put our startupTime
+            int serverStartTime = (int) (session.getKeycloakSessionFactory().getServerStartupTimestamp() / 1000);
+
+            existingClusterStartTime = (Integer) crossDCAwareCacheFactory.getCache().putIfAbsent(InfinispanClusterProvider.CLUSTER_STARTUP_TIME_KEY, serverStartTime);
+            if (existingClusterStartTime == null) {
+                logger.debugf("Initialized cluster startup time to %s", Time.toDate(serverStartTime).toString());
+                return serverStartTime;
+            } else {
+                logger.debugf("Loaded cluster startup time: %s", Time.toDate(existingClusterStartTime).toString());
+                return existingClusterStartTime;
+            }
+        }
+    }
+
+
+
     @Override
     public void init(Config.Scope config) {
     }
@@ -167,34 +220,4 @@ public class InfinispanClusterProviderFactory implements ClusterProviderFactory
     }
 
 
-    <T> void registerListener(String taskKey, ClusterListener task) {
-        listeners.put(taskKey, task);
-    }
-
-    @Listener
-    public class CacheEntryListener {
-
-        @CacheEntryCreated
-        public void cacheEntryCreated(CacheEntryCreatedEvent<String, Object> event) {
-            if (!event.isPre()) {
-                trigger(event.getKey(), event.getValue());
-            }
-        }
-
-        @CacheEntryModified
-        public void cacheEntryModified(CacheEntryModifiedEvent<String, Object> event) {
-            if (!event.isPre()) {
-                trigger(event.getKey(), event.getValue());
-            }
-        }
-
-        private void trigger(String key, Object value) {
-            ClusterListener task = listeners.get(key);
-            if (task != null) {
-                ClusterEvent event = (ClusterEvent) value;
-                task.run(event);
-            }
-        }
-    }
-
 }

model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanNotificationsManager.java

@@ -0,0 +1,204 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.cluster.infinispan;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.infinispan.Cache;
+import org.infinispan.client.hotrod.RemoteCache;
+import org.infinispan.client.hotrod.annotation.ClientCacheEntryCreated;
+import org.infinispan.client.hotrod.annotation.ClientCacheEntryModified;
+import org.infinispan.client.hotrod.annotation.ClientListener;
+import org.infinispan.client.hotrod.event.ClientCacheEntryCreatedEvent;
+import org.infinispan.client.hotrod.event.ClientCacheEntryModifiedEvent;
+import org.infinispan.client.hotrod.event.ClientEvent;
+import org.infinispan.context.Flag;
+import org.infinispan.marshall.core.MarshalledEntry;
+import org.infinispan.notifications.Listener;
+import org.infinispan.notifications.cachelistener.annotation.CacheEntryCreated;
+import org.infinispan.notifications.cachelistener.annotation.CacheEntryModified;
+import org.infinispan.notifications.cachelistener.event.CacheEntryCreatedEvent;
+import org.infinispan.notifications.cachelistener.event.CacheEntryModifiedEvent;
+import org.infinispan.persistence.manager.PersistenceManager;
+import org.infinispan.persistence.remote.RemoteStore;
+import org.infinispan.remoting.transport.Transport;
+import org.jboss.logging.Logger;
+import org.keycloak.cluster.ClusterEvent;
+import org.keycloak.cluster.ClusterListener;
+import org.keycloak.cluster.ClusterProvider;
+import org.keycloak.common.util.HostUtils;
+import org.keycloak.common.util.MultivaluedHashMap;
+
+/**
+ * Impl for sending infinispan messages across cluster and listening to them
+ *
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class InfinispanNotificationsManager {
+
+    protected static final Logger logger = Logger.getLogger(InfinispanNotificationsManager.class);
+
+    private final MultivaluedHashMap<String, ClusterListener> listeners = new MultivaluedHashMap<>();
+
+    private final Cache<String, Serializable> workCache;
+
+    private final String myAddress;
+
+
+    protected InfinispanNotificationsManager(Cache<String, Serializable> workCache, String myAddress) {
+        this.workCache = workCache;
+        this.myAddress = myAddress;
+    }
+
+
+    // Create and init manager including all listeners etc
+    public static InfinispanNotificationsManager create(Cache<String, Serializable> workCache, String myAddress, Set<RemoteStore> remoteStores) {
+        InfinispanNotificationsManager manager = new InfinispanNotificationsManager(workCache, myAddress);
+
+        // We need CacheEntryListener just if we don't have remoteStore. With remoteStore will be all cluster nodes notified anyway from HotRod listener
+        if (remoteStores.isEmpty()) {
+            workCache.addListener(manager.new CacheEntryListener());
+
+            logger.debugf("Added listener for infinispan cache: %s", workCache.getName());
+        } else {
+            for (RemoteStore remoteStore : remoteStores) {
+                RemoteCache<Object, Object> remoteCache = remoteStore.getRemoteCache();
+                remoteCache.addClientListener(manager.new HotRodListener(remoteCache));
+
+                logger.debugf("Added listener for HotRod remoteStore cache: %s", remoteCache.getName());
+            }
+        }
+
+        return manager;
+    }
+
+
+    void registerListener(String taskKey, ClusterListener task) {
+        listeners.add(taskKey, task);
+    }
+
+
+    void notify(String taskKey, ClusterEvent event, boolean ignoreSender) {
+        WrapperClusterEvent wrappedEvent = new WrapperClusterEvent();
+        wrappedEvent.setDelegateEvent(event);
+        wrappedEvent.setIgnoreSender(ignoreSender);
+        wrappedEvent.setSender(myAddress);
+
+        if (logger.isTraceEnabled()) {
+            logger.tracef("Sending event %s: %s", taskKey, event);
+        }
+
+        // Put the value to the cache to notify listeners on all the nodes
+        workCache.getAdvancedCache().withFlags(Flag.IGNORE_RETURN_VALUES)
+                .put(taskKey, wrappedEvent, 120, TimeUnit.SECONDS);
+    }
+
+
+    @Listener(observation = Listener.Observation.POST)
+    public class CacheEntryListener {
+
+        @CacheEntryCreated
+        public void cacheEntryCreated(CacheEntryCreatedEvent<String, Serializable> event) {
+            eventReceived(event.getKey(), event.getValue());
+        }
+
+        @CacheEntryModified
+        public void cacheEntryModified(CacheEntryModifiedEvent<String, Serializable> event) {
+            eventReceived(event.getKey(), event.getValue());
+        }
+    }
+
+
+    @ClientListener
+    public class HotRodListener {
+
+        private final RemoteCache<Object, Object> remoteCache;
+
+        public HotRodListener(RemoteCache<Object, Object> remoteCache) {
+            this.remoteCache = remoteCache;
+        }
+
+
+        @ClientCacheEntryCreated
+        public void created(ClientCacheEntryCreatedEvent event) {
+            String key = event.getKey().toString();
+            hotrodEventReceived(key);
+        }
+
+
+        @ClientCacheEntryModified
+        public void updated(ClientCacheEntryModifiedEvent event) {
+            String key = event.getKey().toString();
+            hotrodEventReceived(key);
+        }
+
+        private void hotrodEventReceived(String key) {
+            // TODO: Look at CacheEventConverter stuff to possibly include value in the event and avoid additional remoteCache request
+            Object value = remoteCache.get(key);
+
+            Serializable rawValue;
+            if (value instanceof MarshalledEntry) {
+                Object rw = ((MarshalledEntry)value).getValue();
+                rawValue = (Serializable) rw;
+            } else {
+                rawValue = (Serializable) value;
+            }
+
+
+            eventReceived(key, rawValue);
+        }
+
+    }
+
+    private void eventReceived(String key, Serializable obj) {
+        if (!(obj instanceof WrapperClusterEvent)) {
+            return;
+        }
+
+        WrapperClusterEvent event = (WrapperClusterEvent) obj;
+
+        if (event.isIgnoreSender()) {
+            if (this.myAddress.equals(event.getSender())) {
+                return;
+            }
+        }
+
+        if (logger.isTraceEnabled()) {
+            logger.tracef("Received event %s: %s", key, event);
+        }
+
+        ClusterEvent wrappedEvent = event.getDelegateEvent();
+
+        List<ClusterListener> myListeners = listeners.get(key);
+        if (myListeners != null) {
+            for (ClusterListener listener : myListeners) {
+                listener.eventReceived(wrappedEvent);
+            }
+        }
+
+        myListeners = listeners.get(ClusterProvider.ALL);
+        if (myListeners != null) {
+            for (ClusterListener listener : myListeners) {
+                listener.eventReceived(wrappedEvent);
+            }
+        }
+    }
+}

model/infinispan/src/main/java/org/keycloak/cluster/infinispan/KeycloakHotRodMarshallerFactory.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.cluster.infinispan;
+
+import org.infinispan.commons.marshall.jboss.GenericJBossMarshaller;
+
+/**
+ * Needed on Wildfly, so that remoteStore (hotRod client) can find our classes
+ *
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class KeycloakHotRodMarshallerFactory {
+
+    public static GenericJBossMarshaller getInstance() {
+        return new GenericJBossMarshaller(KeycloakHotRodMarshallerFactory.class.getClassLoader());
+    }
+
+}

model/infinispan/src/main/java/org/keycloak/cluster/infinispan/WrapperClusterEvent.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.cluster.infinispan;
+
+import org.keycloak.cluster.ClusterEvent;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class WrapperClusterEvent implements ClusterEvent {
+
+    private String sender; // will be null in non-clustered environment
+    private boolean ignoreSender;
+    private ClusterEvent delegateEvent;
+
+    public String getSender() {
+        return sender;
+    }
+
+    public void setSender(String sender) {
+        this.sender = sender;
+    }
+
+    public boolean isIgnoreSender() {
+        return ignoreSender;
+    }
+
+    public void setIgnoreSender(boolean ignoreSender) {
+        this.ignoreSender = ignoreSender;
+    }
+
+    public ClusterEvent getDelegateEvent() {
+        return delegateEvent;
+    }
+
+    public void setDelegateEvent(ClusterEvent delegateEvent) {
+        this.delegateEvent = delegateEvent;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("WrapperClusterEvent [ sender=%s, delegateEvent=%s ]", sender, delegateEvent.toString());
+    }
+}

model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java

@@ -27,11 +27,14 @@ import org.infinispan.eviction.EvictionStrategy;
 import org.infinispan.eviction.EvictionType;
 import org.infinispan.manager.DefaultCacheManager;
 import org.infinispan.manager.EmbeddedCacheManager;
+import org.infinispan.persistence.remote.configuration.ExhaustedAction;
+import org.infinispan.persistence.remote.configuration.RemoteStoreConfigurationBuilder;
 import org.infinispan.transaction.LockingMode;
 import org.infinispan.transaction.TransactionMode;
 import org.infinispan.transaction.lookup.DummyTransactionManagerLookup;
 import org.jboss.logging.Logger;
 import org.keycloak.Config;
+import org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
 
@@ -126,7 +129,7 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
         GlobalConfigurationBuilder gcb = new GlobalConfigurationBuilder();
 
         boolean clustered = config.getBoolean("clustered", false);
-        boolean async = config.getBoolean("async", true);
+        boolean async = config.getBoolean("async", false);
         boolean allowDuplicateJMXDomains = config.getBoolean("allowDuplicateJMXDomains", true);
 
         if (clustered) {
@@ -139,14 +142,11 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
 
         logger.debug("Started embedded Infinispan cache container");
 
-        ConfigurationBuilder invalidationConfigBuilder = new ConfigurationBuilder();
+        ConfigurationBuilder modelCacheConfigBuilder = new ConfigurationBuilder();
-        if (clustered) {
+        Configuration modelCacheConfiguration = modelCacheConfigBuilder.build();
-            invalidationConfigBuilder.clustering().cacheMode(async ? CacheMode.INVALIDATION_ASYNC : CacheMode.INVALIDATION_SYNC);
-        }
-        Configuration invalidationCacheConfiguration = invalidationConfigBuilder.build();
 
-        cacheManager.defineConfiguration(InfinispanConnectionProvider.REALM_CACHE_NAME, invalidationCacheConfiguration);
+        cacheManager.defineConfiguration(InfinispanConnectionProvider.REALM_CACHE_NAME, modelCacheConfiguration);
-        cacheManager.defineConfiguration(InfinispanConnectionProvider.USER_CACHE_NAME, invalidationCacheConfiguration);
+        cacheManager.defineConfiguration(InfinispanConnectionProvider.USER_CACHE_NAME, modelCacheConfiguration);
 
         ConfigurationBuilder sessionConfigBuilder = new ConfigurationBuilder();
         if (clustered) {
@@ -174,8 +174,14 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
         if (clustered) {
             replicationConfigBuilder.clustering().cacheMode(async ? CacheMode.REPL_ASYNC : CacheMode.REPL_SYNC);
         }
-        Configuration replicationCacheConfiguration = replicationConfigBuilder.build();
+
-        cacheManager.defineConfiguration(InfinispanConnectionProvider.WORK_CACHE_NAME, replicationCacheConfiguration);
+        boolean jdgEnabled = config.getBoolean("remoteStoreEnabled");
+        if (jdgEnabled) {
+            configureRemoteCacheStore(replicationConfigBuilder, async);
+        }
+
+        Configuration replicationEvictionCacheConfiguration = replicationConfigBuilder.build();
+        cacheManager.defineConfiguration(InfinispanConnectionProvider.WORK_CACHE_NAME, replicationEvictionCacheConfiguration);
 
         ConfigurationBuilder counterConfigBuilder = new ConfigurationBuilder();
         counterConfigBuilder.invocationBatching().enable()
@@ -211,6 +217,34 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
         return cb.build();
     }
 
+    // Used for cross-data centers scenario. Usually integration with external JDG server, which itself handles communication between DCs.
+    private void configureRemoteCacheStore(ConfigurationBuilder builder, boolean async) {
+        String jdgServer = config.get("remoteStoreServer", "localhost");
+        Integer jdgPort = config.getInt("remoteStorePort", 11222);
+
+        builder.persistence()
+                .passivation(false)
+                .addStore(RemoteStoreConfigurationBuilder.class)
+                    .fetchPersistentState(false)
+                    .ignoreModifications(false)
+                    .purgeOnStartup(false)
+                    .preload(false)
+                    .shared(true)
+                    .remoteCacheName(InfinispanConnectionProvider.WORK_CACHE_NAME)
+                    .rawValues(true)
+                    .forceReturnValues(false)
+                    .marshaller(KeycloakHotRodMarshallerFactory.class.getName())
+                    .addServer()
+                        .host(jdgServer)
+                        .port(jdgPort)
+//                  .connectionPool()
+//                      .maxActive(100)
+//                      .exhaustedAction(ExhaustedAction.CREATE_NEW)
+                    .async()
+                        .enabled(async);
+
+    }
+
     protected Configuration getKeysCacheConfig() {
         ConfigurationBuilder cb = new ConfigurationBuilder();
         cb.eviction().strategy(EvictionStrategy.LRU).type(EvictionType.COUNT).size(InfinispanConnectionProvider.KEYS_CACHE_DEFAULT_MAX);

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/CacheManager.java

@@ -1,14 +1,13 @@
 package org.keycloak.models.cache.infinispan;
 
 import org.infinispan.Cache;
-import org.infinispan.notifications.cachelistener.annotation.CacheEntriesEvicted;
-import org.infinispan.notifications.cachelistener.annotation.CacheEntryInvalidated;
-import org.infinispan.notifications.cachelistener.event.CacheEntriesEvictedEvent;
-import org.infinispan.notifications.cachelistener.event.CacheEntryInvalidatedEvent;
 import org.jboss.logging.Logger;
-import org.keycloak.models.cache.infinispan.entities.AbstractRevisioned;
+import org.keycloak.cluster.ClusterProvider;
+import org.keycloak.models.cache.infinispan.events.InvalidationEvent;
+import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.cache.infinispan.entities.Revisioned;
 
+import java.util.Collection;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map;
@@ -55,7 +54,7 @@ import java.util.function.Predicate;
  * @version $Revision: 1 $
  */
 public abstract class CacheManager {
-    protected static final Logger logger = Logger.getLogger(CacheManager.class);
+
     protected final Cache<String, Long> revisions;
     protected final Cache<String, Revisioned> cache;
     protected final UpdateCounter counter = new UpdateCounter();
@@ -63,9 +62,10 @@ public abstract class CacheManager {
     public CacheManager(Cache<String, Revisioned> cache, Cache<String, Long> revisions) {
         this.cache = cache;
         this.revisions = revisions;
-        this.cache.addListener(this);
     }
 
+    protected abstract Logger getLogger();
+
     public Cache<String, Revisioned> getCache() {
         return cache;
     }
@@ -79,10 +79,7 @@ public abstract class CacheManager {
         if (revision == null) {
             revision = counter.current();
         }
-        // if you do cache.remove() on node 1 and the entry doesn't exist on node 2, node 2 never receives a invalidation event
+
-        // so, we do this to force this.
-        String invalidationKey = "invalidation.key" + id;
-        cache.putForExternalRead(invalidationKey, new AbstractRevisioned(-1L, invalidationKey));
         return revision;
     }
 
@@ -101,12 +98,16 @@ public abstract class CacheManager {
         }
         Long rev = revisions.get(id);
         if (rev == null) {
-            RealmCacheManager.logger.tracev("get() missing rev");
+            if (getLogger().isTraceEnabled()) {
+                getLogger().tracev("get() missing rev {0}", id);
+            }
             return null;
         }
         long oRev = o.getRevision() == null ? -1L : o.getRevision().longValue();
         if (rev > oRev) {
-            RealmCacheManager.logger.tracev("get() rev: {0} o.rev: {1}", rev.longValue(), oRev);
+            if (getLogger().isTraceEnabled()) {
+                getLogger().tracev("get() rev: {0} o.rev: {1}", rev.longValue(), oRev);
+            }
             return null;
         }
         return o != null && type.isInstance(o) ? type.cast(o) : null;
@@ -114,9 +115,11 @@ public abstract class CacheManager {
 
     public Object invalidateObject(String id) {
         Revisioned removed = (Revisioned)cache.remove(id);
-        // if you do cache.remove() on node 1 and the entry doesn't exist on node 2, node 2 never receives a invalidation event
+
-        // so, we do this to force the event.
+        if (getLogger().isTraceEnabled()) {
-        cache.remove("invalidation.key" + id);
+            getLogger().tracef("Removed key='%s', value='%s' from cache", id, removed);
+        }
+
         bumpVersion(id);
         return removed;
     }
@@ -137,37 +140,35 @@ public abstract class CacheManager {
             //revisions.getAdvancedCache().lock(id);
             Long rev = revisions.get(id);
             if (rev == null) {
-                if (id.endsWith("realm.clients")) RealmCacheManager.logger.trace("addRevisioned rev == null realm.clients");
                 rev = counter.current();
                 revisions.put(id, rev);
             }
             revisions.startBatch();
             if (!revisions.getAdvancedCache().lock(id)) {
-                RealmCacheManager.logger.trace("Could not obtain version lock");
+                if (getLogger().isTraceEnabled()) {
+                    getLogger().tracev("Could not obtain version lock: {0}", id);
+                }
                 return;
             }
             rev = revisions.get(id);
             if (rev == null) {
-                if (id.endsWith("realm.clients")) RealmCacheManager.logger.trace("addRevisioned rev2 == null realm.clients");
                 return;
             }
             if (rev > startupRevision) { // revision is ahead transaction start. Other transaction updated in the meantime. Don't cache
-                if (RealmCacheManager.logger.isTraceEnabled()) {
+                if (getLogger().isTraceEnabled()) {
-                    RealmCacheManager.logger.tracev("Skipped cache. Current revision {0}, Transaction start revision {1}", object.getRevision(), startupRevision);
+                    getLogger().tracev("Skipped cache. Current revision {0}, Transaction start revision {1}", object.getRevision(), startupRevision);
                 }
                 return;
             }
             if (rev.equals(object.getRevision())) {
-                if (id.endsWith("realm.clients")) RealmCacheManager.logger.tracev("adding Object.revision {0} rev {1}", object.getRevision(), rev);
                 cache.putForExternalRead(id, object);
                 return;
             }
             if (rev > object.getRevision()) { // revision is ahead, don't cache
-                if (id.endsWith("realm.clients")) RealmCacheManager.logger.trace("addRevisioned revision is ahead realm.clients");
+                if (getLogger().isTraceEnabled()) getLogger().tracev("Skipped cache. Object revision {0}, Cache revision {1}", object.getRevision(), rev);
                 return;
             }
             // revisions cache has a lower value than the object.revision, so update revision and add it to cache
-            if (id.endsWith("realm.clients")) RealmCacheManager.logger.tracev("adding Object.revision {0} rev {1}", object.getRevision(), rev);
             revisions.put(id, object.getRevision());
             if (lifespan < 0) cache.putForExternalRead(id, object);
             else cache.putForExternalRead(id, object, lifespan, TimeUnit.MILLISECONDS);
@@ -196,63 +197,36 @@ public abstract class CacheManager {
                 .filter(predicate).iterator();
     }
 
-    @CacheEntryInvalidated
-    public void cacheInvalidated(CacheEntryInvalidatedEvent<String, Object> event) {
-        if (event.isPre()) {
-            String key = event.getKey();
-            if (key.startsWith("invalidation.key")) {
-                // if you do cache.remove() on node 1 and the entry doesn't exist on node 2, node 2 never receives a invalidation event
-                // so, we do this to force this.
-                String bump = key.substring("invalidation.key".length());
-                RealmCacheManager.logger.tracev("bumping invalidation key {0}", bump);
-                bumpVersion(bump);
-                return;
-            }
 
-        } else {
+    public void sendInvalidationEvents(KeycloakSession session, Collection<InvalidationEvent> invalidationEvents) {
-        //if (!event.isPre()) {
+        ClusterProvider clusterProvider = session.getProvider(ClusterProvider.class);
-            String key = event.getKey();
+
-            if (key.startsWith("invalidation.key")) {
+        // Maybe add InvalidationEvent, which will be collection of all invalidationEvents? That will reduce cluster traffic even more.
-                // if you do cache.remove() on node 1 and the entry doesn't exist on node 2, node 2 never receives a invalidation event
+        for (InvalidationEvent event : invalidationEvents) {
-                // so, we do this to force this.
+            clusterProvider.notify(generateEventId(event), event, true);
-                String bump = key.substring("invalidation.key".length());
-                bumpVersion(bump);
-                RealmCacheManager.logger.tracev("bumping invalidation key {0}", bump);
-                return;
-            }
-            bumpVersion(key);
-            Object object = event.getValue();
-            if (object != null) {
-                bumpVersion(key);
-                Predicate<Map.Entry<String, Revisioned>> predicate = getInvalidationPredicate(object);
-                if (predicate != null) runEvictions(predicate);
-                RealmCacheManager.logger.tracev("invalidating: {0}" + object.getClass().getName());
-            }
         }
     }
 
-    @CacheEntriesEvicted
+    protected String generateEventId(InvalidationEvent event) {
-    public void cacheEvicted(CacheEntriesEvictedEvent<String, Object> event) {
+        return new StringBuilder(event.getId())
-        if (!event.isPre())
+                .append("_")
-        for (Map.Entry<String, Object> entry : event.getEntries().entrySet()) {
+                .append(event.hashCode())
-            Object object = entry.getValue();
+                .toString();
-            bumpVersion(entry.getKey());
+    }
-            if (object == null) continue;
+
-            RealmCacheManager.logger.tracev("evicting: {0}" + object.getClass().getName());
+
-            Predicate<Map.Entry<String, Revisioned>> predicate = getInvalidationPredicate(object);
+    protected void invalidationEventReceived(InvalidationEvent event) {
-            if (predicate != null) runEvictions(predicate);
+        Set<String> invalidations = new HashSet<>();
+
+        addInvalidationsFromEvent(event, invalidations);
+
+        getLogger().debugf("Invalidating %d cache items after received event %s", invalidations.size(), event);
+
+        for (String invalidation : invalidations) {
+            invalidateObject(invalidation);
         }
     }
 
-    public void runEvictions(Predicate<Map.Entry<String, Revisioned>> current) {
+    protected abstract void addInvalidationsFromEvent(InvalidationEvent event, Set<String> invalidations);
-        Set<String> evictions = new HashSet<>();
-        addInvalidations(current, evictions);
-        RealmCacheManager.logger.tracev("running evictions size: {0}", evictions.size());
-        for (String key : evictions) {
-            cache.evict(key);
-            bumpVersion(key);
-        }
-    }
 
-    protected abstract Predicate<Map.Entry<String, Revisioned>> getInvalidationPredicate(Object object);
 }

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/ClientAdapter.java

@@ -52,7 +52,7 @@ public class ClientAdapter implements ClientModel {
 
     private void getDelegateForUpdate() {
         if (updated == null) {
-            cacheSession.registerClientInvalidation(cached.getId());
+            cacheSession.registerClientInvalidation(cached.getId(), cached.getClientId(), cachedRealm.getId());
             updated = cacheSession.getDelegate().getClientById(cached.getId(), cachedRealm);
             if (updated == null) throw new IllegalStateException("Not found in database");
         }
@@ -577,18 +577,12 @@ public class ClientAdapter implements ClientModel {
 
     @Override
     public RoleModel addRole(String name) {
-        getDelegateForUpdate();
+        return cacheSession.addClientRole(getRealm(), this, name);
-        RoleModel role = updated.addRole(name);
-        cacheSession.registerRoleInvalidation(role.getId());
-        return role;
     }
 
     @Override
     public RoleModel addRole(String id, String name) {
-        getDelegateForUpdate();
+        return cacheSession.addClientRole(getRealm(), this, id, name);
-        RoleModel role =  updated.addRole(id, name);
-        cacheSession.registerRoleInvalidation(role.getId());
-        return role;
     }
 
     @Override

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/InfinispanCacheRealmProviderFactory.java

@@ -21,7 +21,6 @@ import org.infinispan.Cache;
 import org.jboss.logging.Logger;
 import org.keycloak.Config;
 import org.keycloak.cluster.ClusterEvent;
-import org.keycloak.cluster.ClusterListener;
 import org.keycloak.cluster.ClusterProvider;
 import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
 import org.keycloak.models.KeycloakSession;
@@ -29,6 +28,7 @@ import org.keycloak.models.KeycloakSessionFactory;
 import org.keycloak.models.cache.CacheRealmProvider;
 import org.keycloak.models.cache.CacheRealmProviderFactory;
 import org.keycloak.models.cache.infinispan.entities.Revisioned;
+import org.keycloak.models.cache.infinispan.events.InvalidationEvent;
 
 /**
  * @author <a href="mailto:bill@burkecentral.com">Bill Burke</a>
@@ -54,14 +54,23 @@ public class InfinispanCacheRealmProviderFactory implements CacheRealmProviderFa
                     Cache<String, Revisioned> cache = session.getProvider(InfinispanConnectionProvider.class).getCache(InfinispanConnectionProvider.REALM_CACHE_NAME);
                     Cache<String, Long> revisions = session.getProvider(InfinispanConnectionProvider.class).getCache(InfinispanConnectionProvider.REALM_REVISIONS_CACHE_NAME);
                     realmCache = new RealmCacheManager(cache, revisions);
+
                     ClusterProvider cluster = session.getProvider(ClusterProvider.class);
-                    cluster.registerListener(REALM_CLEAR_CACHE_EVENTS, new ClusterListener() {
+                    cluster.registerListener(ClusterProvider.ALL, (ClusterEvent event) -> {
-                        @Override
+
-                        public void run(ClusterEvent event) {
+                        if (event instanceof InvalidationEvent) {
-                            realmCache.clear();
+                            InvalidationEvent invalidationEvent = (InvalidationEvent) event;
+                            realmCache.invalidationEventReceived(invalidationEvent);
                         }
                     });
 
+                    cluster.registerListener(REALM_CLEAR_CACHE_EVENTS, (ClusterEvent event) -> {
+
+                        realmCache.clear();
+
+                    });
+
+                    log.debug("Registered cluster listeners");
                 }
             }
         }

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/InfinispanUserCacheProviderFactory.java

@@ -21,7 +21,6 @@ import org.infinispan.Cache;
 import org.jboss.logging.Logger;
 import org.keycloak.Config;
 import org.keycloak.cluster.ClusterEvent;
-import org.keycloak.cluster.ClusterListener;
 import org.keycloak.cluster.ClusterProvider;
 import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
 import org.keycloak.models.KeycloakSession;
@@ -29,6 +28,7 @@ import org.keycloak.models.KeycloakSessionFactory;
 import org.keycloak.models.cache.UserCache;
 import org.keycloak.models.cache.UserCacheProviderFactory;
 import org.keycloak.models.cache.infinispan.entities.Revisioned;
+import org.keycloak.models.cache.infinispan.events.InvalidationEvent;
 
 /**
  * @author <a href="mailto:sthorger@redhat.com">Stian Thorgersen</a>
@@ -55,13 +55,25 @@ public class InfinispanUserCacheProviderFactory implements UserCacheProviderFact
                     Cache<String, Revisioned> cache = session.getProvider(InfinispanConnectionProvider.class).getCache(InfinispanConnectionProvider.USER_CACHE_NAME);
                     Cache<String, Long> revisions = session.getProvider(InfinispanConnectionProvider.class).getCache(InfinispanConnectionProvider.USER_REVISIONS_CACHE_NAME);
                     userCache = new UserCacheManager(cache, revisions);
+
                     ClusterProvider cluster = session.getProvider(ClusterProvider.class);
-                    cluster.registerListener(USER_CLEAR_CACHE_EVENTS, new ClusterListener() {
+
-                        @Override
+                    cluster.registerListener(ClusterProvider.ALL, (ClusterEvent event) -> {
-                        public void run(ClusterEvent event) {
+
-                            userCache.clear();
+                        if (event instanceof InvalidationEvent) {
+                            InvalidationEvent invalidationEvent = (InvalidationEvent) event;
+                            userCache.invalidationEventReceived(invalidationEvent);
                         }
+
                     });
+
+                    cluster.registerListener(USER_CLEAR_CACHE_EVENTS, (ClusterEvent event) -> {
+
+                        userCache.clear();
+
+                    });
+
+                    log.debug("Registered cluster listeners");
                 }
             }
         }

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/RealmAdapter.java

@@ -39,13 +39,8 @@ import org.keycloak.models.UserFederationMapperModel;
 import org.keycloak.models.UserFederationProviderModel;
 import org.keycloak.models.cache.CachedRealmModel;
 import org.keycloak.models.cache.infinispan.entities.CachedRealm;
-import org.keycloak.models.utils.KeycloakModelUtils;
 import org.keycloak.storage.UserStorageProvider;
 
-import java.security.Key;
-import java.security.PrivateKey;
-import java.security.PublicKey;
-import java.security.cert.X509Certificate;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
@@ -75,7 +70,7 @@ public class RealmAdapter implements CachedRealmModel {
     @Override
     public RealmModel getDelegateForUpdate() {
         if (updated == null) {
-            cacheSession.registerRealmInvalidation(cached.getId());
+            cacheSession.registerRealmInvalidation(cached.getId(), cached.getName());
             updated = cacheSession.getDelegate().getRealm(cached.getId());
             if (updated == null) throw new IllegalStateException("Not found in database");
         }
@@ -731,13 +726,6 @@ public class RealmAdapter implements CachedRealmModel {
         updated.setNotBefore(notBefore);
     }
 
-    @Override
-    public boolean removeRoleById(String id) {
-        cacheSession.registerRoleInvalidation(id);
-        getDelegateForUpdate();
-        return updated.removeRoleById(id);
-    }
-
     @Override
     public boolean isEventsEnabled() {
         if (isUpdated()) return updated.isEventsEnabled();
@@ -837,18 +825,12 @@ public class RealmAdapter implements CachedRealmModel {
 
     @Override
     public RoleModel addRole(String name) {
-        getDelegateForUpdate();
+        return cacheSession.addRealmRole(this, name);
-        RoleModel role = updated.addRole(name);
-        cacheSession.registerRoleInvalidation(role.getId());
-        return role;
     }
 
     @Override
     public RoleModel addRole(String id, String name) {
-        getDelegateForUpdate();
+        return cacheSession.addRealmRole(this, id, name);
-        RoleModel role =  updated.addRole(id, name);
-        cacheSession.registerRoleInvalidation(role.getId());
-        return role;
     }
 
     @Override
@@ -1257,12 +1239,6 @@ public class RealmAdapter implements CachedRealmModel {
         return cacheSession.createGroup(this, id, name);
     }
 
-    @Override
-    public void addTopLevelGroup(GroupModel subGroup) {
-        cacheSession.addTopLevelGroup(this, subGroup);
-
-    }
-
     @Override
     public void moveGroup(GroupModel group, GroupModel toParent) {
         cacheSession.moveGroup(this, group, toParent);

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/RealmCacheManager.java

@@ -18,145 +18,88 @@
 package org.keycloak.models.cache.infinispan;
 
 import org.infinispan.Cache;
-import org.infinispan.notifications.Listener;
 import org.jboss.logging.Logger;
-import org.keycloak.models.cache.infinispan.entities.CachedClient;
+import org.keycloak.models.cache.infinispan.events.InvalidationEvent;
-import org.keycloak.models.cache.infinispan.entities.CachedClientTemplate;
-import org.keycloak.models.cache.infinispan.entities.CachedGroup;
-import org.keycloak.models.cache.infinispan.entities.CachedRealm;
-import org.keycloak.models.cache.infinispan.entities.CachedRole;
 import org.keycloak.models.cache.infinispan.entities.Revisioned;
-import org.keycloak.models.cache.infinispan.stream.ClientQueryPredicate;
+import org.keycloak.models.cache.infinispan.events.RealmCacheInvalidationEvent;
-import org.keycloak.models.cache.infinispan.stream.ClientTemplateQueryPredicate;
-import org.keycloak.models.cache.infinispan.stream.GroupQueryPredicate;
 import org.keycloak.models.cache.infinispan.stream.HasRolePredicate;
 import org.keycloak.models.cache.infinispan.stream.InClientPredicate;
 import org.keycloak.models.cache.infinispan.stream.InRealmPredicate;
-import org.keycloak.models.cache.infinispan.stream.RealmQueryPredicate;
 
-import java.util.Map;
 import java.util.Set;
-import java.util.function.Predicate;
 
 /**
  * @author <a href="mailto:sthorger@redhat.com">Stian Thorgersen</a>
  */
-@Listener
 public class RealmCacheManager extends CacheManager {
 
-    protected static final Logger logger = Logger.getLogger(RealmCacheManager.class);
+    private static final Logger logger = Logger.getLogger(RealmCacheManager.class);
+
+    @Override
+    protected Logger getLogger() {
+        return logger;
+    }
 
     public RealmCacheManager(Cache<String, Revisioned> cache, Cache<String, Long> revisions) {
         super(cache, revisions);
     }
 
 
-    public void realmInvalidation(String id, Set<String> invalidations) {
+    public void realmUpdated(String id, String name, Set<String> invalidations) {
-        Predicate<Map.Entry<String, Revisioned>> predicate = getRealmInvalidationPredicate(id);
+        invalidations.add(id);
-        addInvalidations(predicate, invalidations);
+        invalidations.add(RealmCacheSession.getRealmByNameCacheKey(name));
     }
 
-    public Predicate<Map.Entry<String, Revisioned>> getRealmInvalidationPredicate(String id) {
+    public void realmRemoval(String id, String name, Set<String> invalidations) {
-        return RealmQueryPredicate.create().realm(id);
+        realmUpdated(id, name, invalidations);
+
+        addInvalidations(InRealmPredicate.create().realm(id), invalidations);
     }
 
-    public void clientInvalidation(String id, Set<String> invalidations) {
+    public void roleAdded(String roleContainerId, Set<String> invalidations) {
-        addInvalidations(getClientInvalidationPredicate(id), invalidations);
+        invalidations.add(RealmCacheSession.getRolesCacheKey(roleContainerId));
     }
 
-    public Predicate<Map.Entry<String, Revisioned>> getClientInvalidationPredicate(String id) {
+    public void roleUpdated(String roleContainerId, String roleName, Set<String> invalidations) {
-        return ClientQueryPredicate.create().client(id);
+        invalidations.add(RealmCacheSession.getRoleByNameCacheKey(roleContainerId, roleName));
     }
 
-    public void roleInvalidation(String id, Set<String> invalidations) {
+    public void roleRemoval(String id, String roleName, String roleContainerId, Set<String> invalidations) {
-        addInvalidations(getRoleInvalidationPredicate(id), invalidations);
+        invalidations.add(RealmCacheSession.getRolesCacheKey(roleContainerId));
+        invalidations.add(RealmCacheSession.getRoleByNameCacheKey(roleContainerId, roleName));
 
+        addInvalidations(HasRolePredicate.create().role(id), invalidations);
     }
 
-    public Predicate<Map.Entry<String, Revisioned>> getRoleInvalidationPredicate(String id) {
+    public void groupQueriesInvalidations(String realmId, Set<String> invalidations) {
-        return HasRolePredicate.create().role(id);
+        invalidations.add(RealmCacheSession.getGroupsQueryCacheKey(realmId));
+        invalidations.add(RealmCacheSession.getTopGroupsQueryCacheKey(realmId)); // Just easier to always invalidate top-level too. It's not big performance penalty
     }
 
-    public void groupInvalidation(String id, Set<String> invalidations) {
+    public void clientAdded(String realmId, String clientUUID, String clientId, Set<String> invalidations) {
-        addInvalidations(getGroupInvalidationPredicate(id), invalidations);
+        invalidations.add(RealmCacheSession.getRealmClientsQueryCacheKey(realmId));
-
     }
 
-    public Predicate<Map.Entry<String, Revisioned>> getGroupInvalidationPredicate(String id) {
+    public void clientUpdated(String realmId, String clientUuid, String clientId, Set<String> invalidations) {
-        return GroupQueryPredicate.create().group(id);
+        invalidations.add(RealmCacheSession.getClientByClientIdCacheKey(clientId, realmId));
     }
 
-    public void clientTemplateInvalidation(String id, Set<String> invalidations) {
+    // Client roles invalidated separately
-        addInvalidations(getClientTemplateInvalidationPredicate(id), invalidations);
+    public void clientRemoval(String realmId, String clientUUID, String clientId, Set<String> invalidations) {
+        invalidations.add(RealmCacheSession.getRealmClientsQueryCacheKey(realmId));
+        invalidations.add(RealmCacheSession.getClientByClientIdCacheKey(clientId, realmId));
 
+        addInvalidations(InClientPredicate.create().client(clientUUID), invalidations);
     }
 
-    public Predicate<Map.Entry<String, Revisioned>> getClientTemplateInvalidationPredicate(String id) {
-        return ClientTemplateQueryPredicate.create().template(id);
-    }
-
-    public void realmRemoval(String id, Set<String> invalidations) {
-        Predicate<Map.Entry<String, Revisioned>> predicate = getRealmRemovalPredicate(id);
-        addInvalidations(predicate, invalidations);
-    }
-
-    public Predicate<Map.Entry<String, Revisioned>> getRealmRemovalPredicate(String id) {
-        Predicate<Map.Entry<String, Revisioned>> predicate = null;
-        predicate = RealmQueryPredicate.create().realm(id)
-                .or(InRealmPredicate.create().realm(id));
-        return predicate;
-    }
-
-    public void clientAdded(String realmId, String id, Set<String> invalidations) {
-        addInvalidations(getClientAddedPredicate(realmId), invalidations);
-    }
-
-    public Predicate<Map.Entry<String, Revisioned>> getClientAddedPredicate(String realmId) {
-        return ClientQueryPredicate.create().inRealm(realmId);
-    }
-
-    public void clientRemoval(String realmId, String id, Set<String> invalidations) {
-        Predicate<Map.Entry<String, Revisioned>> predicate = null;
-        predicate = getClientRemovalPredicate(realmId, id);
-        addInvalidations(predicate, invalidations);
-    }
-
-    public Predicate<Map.Entry<String, Revisioned>> getClientRemovalPredicate(String realmId, String id) {
-        Predicate<Map.Entry<String, Revisioned>> predicate;
-        predicate = ClientQueryPredicate.create().inRealm(realmId)
-                .or(ClientQueryPredicate.create().client(id))
-                .or(InClientPredicate.create().client(id));
-        return predicate;
-    }
-
-    public void roleRemoval(String id, Set<String> invalidations) {
-        addInvalidations(getRoleRemovalPredicate(id), invalidations);
-
-    }
-
-    public Predicate<Map.Entry<String, Revisioned>> getRoleRemovalPredicate(String id) {
-        return getRoleInvalidationPredicate(id);
-    }
 
     @Override
-    protected Predicate<Map.Entry<String, Revisioned>> getInvalidationPredicate(Object object) {
+    protected void addInvalidationsFromEvent(InvalidationEvent event, Set<String> invalidations) {
-        if (object instanceof CachedRealm) {
+        if (event instanceof RealmCacheInvalidationEvent) {
-            CachedRealm cached = (CachedRealm)object;
+            invalidations.add(event.getId());
-            return getRealmRemovalPredicate(cached.getId());
+
-        } else if (object instanceof CachedClient) {
+            ((RealmCacheInvalidationEvent) event).addInvalidations(this, invalidations);
-            CachedClient cached = (CachedClient)object;
-            Predicate<Map.Entry<String, Revisioned>> predicate = getClientRemovalPredicate(cached.getRealm(), cached.getId());
-            return predicate;
-        } else if (object instanceof CachedRole) {
-            CachedRole cached = (CachedRole)object;
-            return getRoleRemovalPredicate(cached.getId());
-        } else if (object instanceof CachedGroup) {
-            CachedGroup cached = (CachedGroup)object;
-            return getGroupInvalidationPredicate(cached.getId());
-        } else if (object instanceof CachedClientTemplate) {
-            CachedClientTemplate cached = (CachedClientTemplate)object;
-            return getClientTemplateInvalidationPredicate(cached.getId());
         }
-        return null;
     }
+
 }

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/RealmCacheSession.java

@@ -19,6 +19,7 @@ package org.keycloak.models.cache.infinispan;
 
 import org.jboss.logging.Logger;
 import org.keycloak.cluster.ClusterProvider;
+import org.keycloak.models.cache.infinispan.events.InvalidationEvent;
 import org.keycloak.migration.MigrationModel;
 import org.keycloak.models.ClientModel;
 import org.keycloak.models.ClientTemplateModel;
@@ -38,8 +39,22 @@ import org.keycloak.models.cache.infinispan.entities.CachedRealm;
 import org.keycloak.models.cache.infinispan.entities.CachedRealmRole;
 import org.keycloak.models.cache.infinispan.entities.CachedRole;
 import org.keycloak.models.cache.infinispan.entities.ClientListQuery;
+import org.keycloak.models.cache.infinispan.entities.GroupListQuery;
 import org.keycloak.models.cache.infinispan.entities.RealmListQuery;
 import org.keycloak.models.cache.infinispan.entities.RoleListQuery;
+import org.keycloak.models.cache.infinispan.events.ClientAddedEvent;
+import org.keycloak.models.cache.infinispan.events.ClientRemovedEvent;
+import org.keycloak.models.cache.infinispan.events.ClientTemplateEvent;
+import org.keycloak.models.cache.infinispan.events.ClientUpdatedEvent;
+import org.keycloak.models.cache.infinispan.events.GroupAddedEvent;
+import org.keycloak.models.cache.infinispan.events.GroupMovedEvent;
+import org.keycloak.models.cache.infinispan.events.GroupRemovedEvent;
+import org.keycloak.models.cache.infinispan.events.GroupUpdatedEvent;
+import org.keycloak.models.cache.infinispan.events.RealmRemovedEvent;
+import org.keycloak.models.cache.infinispan.events.RealmUpdatedEvent;
+import org.keycloak.models.cache.infinispan.events.RoleAddedEvent;
+import org.keycloak.models.cache.infinispan.events.RoleRemovedEvent;
+import org.keycloak.models.cache.infinispan.events.RoleUpdatedEvent;
 import org.keycloak.models.utils.KeycloakModelUtils;
 
 import java.util.HashMap;
@@ -126,6 +141,7 @@ public class RealmCacheSession implements CacheRealmProvider {
     protected Map<String, GroupAdapter> managedGroups = new HashMap<>();
     protected Set<String> listInvalidations = new HashSet<>();
     protected Set<String> invalidations = new HashSet<>();
+    protected Set<InvalidationEvent> invalidationEvents = new HashSet<>(); // Events to be sent across cluster
 
     protected boolean clearAll;
     protected final long startupRevision;
@@ -150,7 +166,7 @@ public class RealmCacheSession implements CacheRealmProvider {
     public void clear() {
         cache.clear();
         ClusterProvider cluster = session.getProvider(ClusterProvider.class);
-        cluster.notify(InfinispanCacheRealmProviderFactory.REALM_CLEAR_CACHE_EVENTS, new ClearCacheEvent());
+        cluster.notify(InfinispanCacheRealmProviderFactory.REALM_CLEAR_CACHE_EVENTS, new ClearCacheEvent(), true);
     }
 
     @Override
@@ -167,21 +183,19 @@ public class RealmCacheSession implements CacheRealmProvider {
     }
 
     @Override
-    public void registerRealmInvalidation(String id) {
+    public void registerRealmInvalidation(String id, String name) {
-        invalidateRealm(id);
+        cache.realmUpdated(id, name, invalidations);
-        cache.realmInvalidation(id, invalidations);
-    }
-
-    private void invalidateRealm(String id) {
-        invalidations.add(id);
         RealmAdapter adapter = managedRealms.get(id);
         if (adapter != null) adapter.invalidateFlag();
+
+        invalidationEvents.add(RealmUpdatedEvent.create(id, name));
     }
 
     @Override
-    public void registerClientInvalidation(String id) {
+    public void registerClientInvalidation(String id, String clientId, String realmId) {
         invalidateClient(id);
-        cache.clientInvalidation(id, invalidations);
+        invalidationEvents.add(ClientUpdatedEvent.create(id, clientId, realmId));
+        cache.clientUpdated(realmId, id, clientId, invalidations);
     }
 
     private void invalidateClient(String id) {
@@ -193,7 +207,9 @@ public class RealmCacheSession implements CacheRealmProvider {
     @Override
     public void registerClientTemplateInvalidation(String id) {
         invalidateClientTemplate(id);
-        cache.clientTemplateInvalidation(id, invalidations);
+        // Note: Adding/Removing client template is supposed to invalidate CachedRealm as well, so the list of clientTemplates is invalidated.
+        // But separate RealmUpdatedEvent will be sent for it. So ClientTemplateEvent don't need to take care of it.
+        invalidationEvents.add(ClientTemplateEvent.create(id));
     }
 
     private void invalidateClientTemplate(String id) {
@@ -203,14 +219,15 @@ public class RealmCacheSession implements CacheRealmProvider {
     }
 
     @Override
-    public void registerRoleInvalidation(String id) {
+    public void registerRoleInvalidation(String id, String roleName, String roleContainerId) {
         invalidateRole(id);
-        roleInvalidations(id);
+        cache.roleUpdated(roleContainerId, roleName, invalidations);
+        invalidationEvents.add(RoleUpdatedEvent.create(id, roleName, roleContainerId));
     }
 
-    private void roleInvalidations(String roleId) {
+    private void roleRemovalInvalidations(String roleId, String roleName, String roleContainerId) {
         Set<String> newInvalidations = new HashSet<>();
-        cache.roleInvalidation(roleId, newInvalidations);
+        cache.roleRemoval(roleId, roleName, roleContainerId, newInvalidations);
         invalidations.addAll(newInvalidations);
         // need to make sure that scope and group mapping clients and groups are invalidated
         for (String id : newInvalidations) {
@@ -229,6 +246,11 @@ public class RealmCacheSession implements CacheRealmProvider {
                 clientTemplate.invalidate();
                 continue;
             }
+            RoleAdapter role = managedRoles.get(id);
+            if (role != null) {
+                role.invalidate();
+                continue;
+            }
 
 
         }
@@ -243,10 +265,26 @@ public class RealmCacheSession implements CacheRealmProvider {
         if (adapter != null) adapter.invalidate();
     }
 
+    private void addedRole(String roleId, String roleContainerId) {
+        // this is needed so that a new role that hasn't been committed isn't cached in a query
+        listInvalidations.add(roleContainerId);
+
+        invalidateRole(roleId);
+        cache.roleAdded(roleContainerId, invalidations);
+        invalidationEvents.add(RoleAddedEvent.create(roleId, roleContainerId));
+    }
+
     @Override
     public void registerGroupInvalidation(String id) {
+        invalidateGroup(id, null, false);
+        addGroupEventIfAbsent(GroupUpdatedEvent.create(id));
+    }
+
+    private void invalidateGroup(String id, String realmId, boolean invalidateQueries) {
         invalidateGroup(id);
-        cache.groupInvalidation(id, invalidations);
+        if (invalidateQueries) {
+            cache.groupQueriesInvalidations(realmId, invalidations);
+        }
     }
 
     private void invalidateGroup(String id) {
@@ -259,6 +297,8 @@ public class RealmCacheSession implements CacheRealmProvider {
         for (String id : invalidations) {
             cache.invalidateObject(id);
         }
+
+        cache.sendInvalidationEvents(session, invalidationEvents);
     }
 
     private KeycloakTransaction getPrepareTransaction() {
@@ -358,14 +398,14 @@ public class RealmCacheSession implements CacheRealmProvider {
     @Override
     public RealmModel createRealm(String name) {
         RealmModel realm = getDelegate().createRealm(name);
-        registerRealmInvalidation(realm.getId());
+        registerRealmInvalidation(realm.getId(), realm.getName());
         return realm;
     }
 
     @Override
     public RealmModel createRealm(String id, String name) {
         RealmModel realm =  getDelegate().createRealm(id, name);
-        registerRealmInvalidation(realm.getId());
+        registerRealmInvalidation(realm.getId(), realm.getName());
         return realm;
     }
 
@@ -434,7 +474,7 @@ public class RealmCacheSession implements CacheRealmProvider {
         }
     }
 
-    public String getRealmByNameCacheKey(String name) {
+    static String getRealmByNameCacheKey(String name) {
         return "realm.query.by.name." + name;
     }
 
@@ -457,20 +497,12 @@ public class RealmCacheSession implements CacheRealmProvider {
         RealmModel realm = getRealm(id);
         if (realm == null) return false;
 
-        invalidations.add(getRealmClientsQueryCacheKey(id));
-        invalidations.add(getRealmByNameCacheKey(realm.getName()));
         cache.invalidateObject(id);
-        cache.realmRemoval(id, invalidations);
+        invalidationEvents.add(RealmRemovedEvent.create(id, realm.getName()));
+        cache.realmRemoval(id, realm.getName(), invalidations);
         return getDelegate().removeRealm(id);
     }
 
-    protected void invalidateClient(RealmModel realm, ClientModel client) {
-        invalidateClient(client.getId());
-        invalidations.add(getRealmClientsQueryCacheKey(realm.getId()));
-        invalidations.add(getClientByClientIdCacheKey(client.getClientId(), realm));
-        listInvalidations.add(realm.getId());
-    }
-
 
     @Override
     public ClientModel addClient(RealmModel realm, String clientId) {
@@ -486,30 +518,32 @@ public class RealmCacheSession implements CacheRealmProvider {
 
     private ClientModel addedClient(RealmModel realm, ClientModel client) {
         logger.trace("added Client.....");
-        // need to invalidate realm client query cache every time as it may not be loaded on this node, but loaded on another
+
-        invalidateClient(realm, client);
+        invalidateClient(client.getId());
-        cache.clientAdded(realm.getId(), client.getId(), invalidations);
+        // this is needed so that a client that hasn't been committed isn't cached in a query
-        // this is needed so that a new client that hasn't been committed isn't cached in a query
         listInvalidations.add(realm.getId());
+
+        invalidationEvents.add(ClientAddedEvent.create(client.getId(), client.getClientId(), realm.getId()));
+        cache.clientAdded(realm.getId(), client.getId(), client.getClientId(), invalidations);
         return client;
     }
 
-    private String getRealmClientsQueryCacheKey(String realm) {
+    static String getRealmClientsQueryCacheKey(String realm) {
         return realm + REALM_CLIENTS_QUERY_SUFFIX;
     }
 
-    private String getGroupsQueryCacheKey(String realm) {
+    static String getGroupsQueryCacheKey(String realm) {
         return realm + ".groups";
     }
 
-    private String getTopGroupsQueryCacheKey(String realm) {
+    static String getTopGroupsQueryCacheKey(String realm) {
         return realm + ".top.groups";
     }
 
-    private String getRolesCacheKey(String container) {
+    static String getRolesCacheKey(String container) {
         return container + ROLES_QUERY_SUFFIX;
     }
-    private String getRoleByNameCacheKey(String container, String name) {
+    static String getRoleByNameCacheKey(String container, String name) {
         return container + "." + name + ROLES_QUERY_SUFFIX;
     }
 
@@ -541,6 +575,7 @@ public class RealmCacheSession implements CacheRealmProvider {
         for (String id : query.getClients()) {
             ClientModel client = session.realms().getClientById(id, realm);
             if (client == null) {
+                // TODO: Handle with cluster invalidations too
                 invalidations.add(cacheKey);
                 return getDelegate().getClients(realm);
             }
@@ -554,12 +589,16 @@ public class RealmCacheSession implements CacheRealmProvider {
     public boolean removeClient(String id, RealmModel realm) {
         ClientModel client = getClientById(id, realm);
         if (client == null) return false;
-        // need to invalidate realm client query cache every time client list is changed
+
-        invalidateClient(realm, client);
+        invalidateClient(client.getId());
-        cache.clientRemoval(realm.getId(), id, invalidations);
+        // this is needed so that a client that hasn't been committed isn't cached in a query
+        listInvalidations.add(realm.getId());
+
+        invalidationEvents.add(ClientRemovedEvent.create(client));
+        cache.clientRemoval(realm.getId(), id, client.getClientId(), invalidations);
+
         for (RoleModel role : client.getRoles()) {
-            String roleId = role.getId();
+            roleRemovalInvalidations(role.getId(), role.getName(), client.getId());
-            roleInvalidations(roleId);
         }
         return getDelegate().removeClient(id, realm);
     }
@@ -577,11 +616,8 @@ public class RealmCacheSession implements CacheRealmProvider {
 
     @Override
     public RoleModel addRealmRole(RealmModel realm, String id, String name) {
-        invalidations.add(getRolesCacheKey(realm.getId()));
-        // this is needed so that a new role that hasn't been committed isn't cached in a query
-        listInvalidations.add(realm.getId());
         RoleModel role = getDelegate().addRealmRole(realm, name);
-        invalidations.add(role.getId());
+        addedRole(role.getId(), realm.getId());
         return role;
     }
 
@@ -664,11 +700,8 @@ public class RealmCacheSession implements CacheRealmProvider {
 
     @Override
     public RoleModel addClientRole(RealmModel realm, ClientModel client, String id, String name) {
-        invalidations.add(getRolesCacheKey(client.getId()));
-        // this is needed so that a new role that hasn't been committed isn't cached in a query
-        listInvalidations.add(client.getId());
         RoleModel role = getDelegate().addClientRole(realm, client, id, name);
-        invalidateRole(role.getId());
+        addedRole(role.getId(), client.getId());
         return role;
     }
 
@@ -734,10 +767,12 @@ public class RealmCacheSession implements CacheRealmProvider {
 
     @Override
     public boolean removeRole(RealmModel realm, RoleModel role) {
-        invalidations.add(getRolesCacheKey(role.getContainer().getId()));
-        invalidations.add(getRoleByNameCacheKey(role.getContainer().getId(), role.getName()));
         listInvalidations.add(role.getContainer().getId());
-        registerRoleInvalidation(role.getId());
+
+        invalidateRole(role.getId());
+        invalidationEvents.add(RoleRemovedEvent.create(role.getId(), role.getName(), role.getContainer().getId()));
+        roleRemovalInvalidations(role.getId(), role.getName(), role.getContainer().getId());
+
         return getDelegate().removeRole(realm, role);
     }
 
@@ -797,8 +832,11 @@ public class RealmCacheSession implements CacheRealmProvider {
 
     @Override
     public void moveGroup(RealmModel realm, GroupModel group, GroupModel toParent) {
-        registerGroupInvalidation(group.getId());
+        invalidateGroup(group.getId(), realm.getId(), true);
-        if (toParent != null) registerGroupInvalidation(toParent.getId());
+        if (toParent != null) invalidateGroup(group.getId(), realm.getId(), false); // Queries already invalidated
+        listInvalidations.add(realm.getId());
+
+        invalidationEvents.add(GroupMovedEvent.create(group, toParent, realm.getId()));
         getDelegate().moveGroup(realm, group, toParent);
     }
 
@@ -876,14 +914,15 @@ public class RealmCacheSession implements CacheRealmProvider {
 
     @Override
     public boolean removeGroup(RealmModel realm, GroupModel group) {
-        registerGroupInvalidation(group.getId());
+        invalidateGroup(group.getId(), realm.getId(), true);
         listInvalidations.add(realm.getId());
-        invalidations.add(getGroupsQueryCacheKey(realm.getId()));
+        cache.groupQueriesInvalidations(realm.getId(), invalidations);
-        if (group.getParentId() == null) {
+        if (group.getParentId() != null) {
-            invalidations.add(getTopGroupsQueryCacheKey(realm.getId()));
+            invalidateGroup(group.getParentId(), realm.getId(), false); // Queries already invalidated
-        } else {
-            registerGroupInvalidation(group.getParentId());
         }
+
+        invalidationEvents.add(GroupRemovedEvent.create(group, realm.getId()));
+
         return getDelegate().removeGroup(realm, group);
     }
 
@@ -893,11 +932,11 @@ public class RealmCacheSession implements CacheRealmProvider {
         return groupAdded(realm, group);
     }
 
-    public GroupModel groupAdded(RealmModel realm, GroupModel group) {
+    private GroupModel groupAdded(RealmModel realm, GroupModel group) {
         listInvalidations.add(realm.getId());
-        invalidations.add(getGroupsQueryCacheKey(realm.getId()));
+        cache.groupQueriesInvalidations(realm.getId(), invalidations);
-        invalidations.add(getTopGroupsQueryCacheKey(realm.getId()));
         invalidations.add(group.getId());
+        invalidationEvents.add(GroupAddedEvent.create(group.getId(), realm.getId()));
         return group;
     }
 
@@ -909,15 +948,32 @@ public class RealmCacheSession implements CacheRealmProvider {
 
     @Override
     public void addTopLevelGroup(RealmModel realm, GroupModel subGroup) {
-        invalidations.add(getTopGroupsQueryCacheKey(realm.getId()));
+        invalidateGroup(subGroup.getId(), realm.getId(), true);
-        invalidations.add(subGroup.getId());
         if (subGroup.getParentId() != null) {
-            registerGroupInvalidation(subGroup.getParentId());
+            invalidateGroup(subGroup.getParentId(), realm.getId(), false); // Queries already invalidated
         }
+
+        addGroupEventIfAbsent(GroupMovedEvent.create(subGroup, null, realm.getId()));
+
         getDelegate().addTopLevelGroup(realm, subGroup);
 
     }
 
+    private void addGroupEventIfAbsent(InvalidationEvent eventToAdd) {
+        String groupId = eventToAdd.getId();
+
+        // Check if we have existing event with bigger priority
+        boolean eventAlreadyExists = invalidationEvents.stream().filter((InvalidationEvent event) -> {
+
+            return (event.getId().equals(groupId)) && (event instanceof GroupAddedEvent || event instanceof GroupMovedEvent || event instanceof GroupRemovedEvent);
+
+        }).findFirst().isPresent();
+
+        if (!eventAlreadyExists) {
+            invalidationEvents.add(eventToAdd);
+        }
+    }
+
     @Override
     public ClientModel getClientById(String id, RealmModel realm) {
         CachedClient cached = cache.get(id, CachedClient.class);
@@ -948,7 +1004,7 @@ public class RealmCacheSession implements CacheRealmProvider {
 
     @Override
     public ClientModel getClientByClientId(String clientId, RealmModel realm) {
-        String cacheKey = getClientByClientIdCacheKey(clientId, realm);
+        String cacheKey = getClientByClientIdCacheKey(clientId, realm.getId());
         ClientListQuery query = cache.get(cacheKey, ClientListQuery.class);
         String id = null;
 
@@ -976,8 +1032,8 @@ public class RealmCacheSession implements CacheRealmProvider {
         return getClientById(id, realm);
     }
 
-    public String getClientByClientIdCacheKey(String clientId, RealmModel realm) {
+    static String getClientByClientIdCacheKey(String clientId, String realmId) {
-        return realm.getId() + ".client.query.by.clientId." + clientId;
+        return realmId + ".client.query.by.clientId." + clientId;
     }
 
     @Override

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/RoleAdapter.java

@@ -47,7 +47,7 @@ public class RoleAdapter implements RoleModel {
 
     protected void getDelegateForUpdate() {
         if (updated == null) {
-            cacheSession.registerRoleInvalidation(cached.getId());
+            cacheSession.registerRoleInvalidation(cached.getId(), cached.getName(), getContainerId());
             updated = cacheSession.getDelegate().getRoleById(cached.getId(), realm);
             if (updated == null) throw new IllegalStateException("Not found in database");
         }

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/UserCacheManager.java

@@ -18,40 +18,94 @@
 package org.keycloak.models.cache.infinispan;
 
 import org.infinispan.Cache;
-import org.infinispan.notifications.Listener;
 import org.jboss.logging.Logger;
+import org.keycloak.models.cache.infinispan.events.InvalidationEvent;
 import org.keycloak.models.cache.infinispan.entities.Revisioned;
+import org.keycloak.models.cache.infinispan.events.UserCacheInvalidationEvent;
 import org.keycloak.models.cache.infinispan.stream.InRealmPredicate;
 
 import java.util.Map;
 import java.util.Set;
-import java.util.function.Predicate;
 
 /**
  * @author <a href="mailto:sthorger@redhat.com">Stian Thorgersen</a>
  */
-@Listener
 public class UserCacheManager extends CacheManager {
 
-    protected static final Logger logger = Logger.getLogger(UserCacheManager.class);
+    private static final Logger logger = Logger.getLogger(UserCacheManager.class);
 
     protected volatile boolean enabled = true;
+
     public UserCacheManager(Cache<String, Revisioned> cache, Cache<String, Long> revisions) {
         super(cache, revisions);
     }
 
+    @Override
+    protected Logger getLogger() {
+        return logger;
+    }
+
     @Override
     public void clear() {
         cache.clear();
         revisions.clear();
     }
 
+
+    public void userUpdatedInvalidations(String userId, String username, String email, String realmId, Set<String> invalidations) {
+        invalidations.add(userId);
+        if (email != null) invalidations.add(UserCacheSession.getUserByEmailCacheKey(realmId, email));
+        invalidations.add(UserCacheSession.getUserByUsernameCacheKey(realmId, username));
+    }
+
+    // Fully invalidate user including consents and federatedIdentity links.
+    public void fullUserInvalidation(String userId, String username, String email, String realmId, boolean identityFederationEnabled, Map<String, String> federatedIdentities, Set<String> invalidations) {
+        userUpdatedInvalidations(userId, username, email, realmId, invalidations);
+
+        if (identityFederationEnabled) {
+            // Invalidate all keys for lookup this user by any identityProvider link
+            for (Map.Entry<String, String> socialLink : federatedIdentities.entrySet()) {
+                String fedIdentityCacheKey = UserCacheSession.getUserByFederatedIdentityCacheKey(realmId, socialLink.getKey(), socialLink.getValue());
+                invalidations.add(fedIdentityCacheKey);
+            }
+
+            // Invalidate federationLinks of user
+            invalidations.add(UserCacheSession.getFederatedIdentityLinksCacheKey(userId));
+        }
+
+        // Consents
+        invalidations.add(UserCacheSession.getConsentCacheKey(userId));
+    }
+
+    public void federatedIdentityLinkUpdatedInvalidation(String userId, Set<String> invalidations) {
+        invalidations.add(UserCacheSession.getFederatedIdentityLinksCacheKey(userId));
+    }
+
+    public void federatedIdentityLinkRemovedInvalidation(String userId, String realmId, String identityProviderId, String socialUserId, Set<String> invalidations) {
+        invalidations.add(UserCacheSession.getFederatedIdentityLinksCacheKey(userId));
+        if (identityProviderId != null) {
+            invalidations.add(UserCacheSession.getUserByFederatedIdentityCacheKey(realmId, identityProviderId, socialUserId));
+        }
+    }
+
+    public void consentInvalidation(String userId, Set<String> invalidations) {
+        invalidations.add(UserCacheSession.getConsentCacheKey(userId));
+    }
+
+
     @Override
-    protected Predicate<Map.Entry<String, Revisioned>> getInvalidationPredicate(Object object) {
+    protected void addInvalidationsFromEvent(InvalidationEvent event, Set<String> invalidations) {
-        return null;
+        if (event instanceof UserCacheInvalidationEvent) {
+            ((UserCacheInvalidationEvent) event).addInvalidations(this, invalidations);
+        }
     }
 
     public void invalidateRealmUsers(String realm, Set<String> invalidations) {
-        addInvalidations(InRealmPredicate.create().realm(realm), invalidations);
+        InRealmPredicate inRealmPredicate = getInRealmPredicate(realm);
+        addInvalidations(inRealmPredicate, invalidations);
+    }
+
+    private InRealmPredicate getInRealmPredicate(String realmId) {
+        return InRealmPredicate.create().realm(realmId);
     }
 }

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/UserCacheSession.java

@@ -19,6 +19,7 @@ package org.keycloak.models.cache.infinispan;
 
 import org.jboss.logging.Logger;
 import org.keycloak.cluster.ClusterProvider;
+import org.keycloak.models.cache.infinispan.events.InvalidationEvent;
 import org.keycloak.common.constants.ServiceAccountConstants;
 import org.keycloak.common.util.Time;
 import org.keycloak.component.ComponentModel;
@@ -42,6 +43,12 @@ import org.keycloak.models.cache.infinispan.entities.CachedUser;
 import org.keycloak.models.cache.infinispan.entities.CachedUserConsent;
 import org.keycloak.models.cache.infinispan.entities.CachedUserConsents;
 import org.keycloak.models.cache.infinispan.entities.UserListQuery;
+import org.keycloak.models.cache.infinispan.events.UserCacheRealmInvalidationEvent;
+import org.keycloak.models.cache.infinispan.events.UserConsentsUpdatedEvent;
+import org.keycloak.models.cache.infinispan.events.UserFederationLinkRemovedEvent;
+import org.keycloak.models.cache.infinispan.events.UserFederationLinkUpdatedEvent;
+import org.keycloak.models.cache.infinispan.events.UserFullInvalidationEvent;
+import org.keycloak.models.cache.infinispan.events.UserUpdatedEvent;
 import org.keycloak.storage.StorageId;
 import org.keycloak.storage.UserStorageProvider;
 import org.keycloak.storage.UserStorageProviderModel;
@@ -72,6 +79,7 @@ public class UserCacheSession implements UserCache {
 
     protected Set<String> invalidations = new HashSet<>();
     protected Set<String> realmInvalidations = new HashSet<>();
+    protected Set<InvalidationEvent> invalidationEvents = new HashSet<>(); // Events to be sent across cluster
     protected Map<String, UserModel> managedUsers = new HashMap<>();
 
     public UserCacheSession(UserCacheManager cache, KeycloakSession session) {
@@ -85,7 +93,7 @@ public class UserCacheSession implements UserCache {
     public void clear() {
         cache.clear();
         ClusterProvider cluster = session.getProvider(ClusterProvider.class);
-        cluster.notify(InfinispanUserCacheProviderFactory.USER_CLEAR_CACHE_EVENTS, new ClearCacheEvent());
+        cluster.notify(InfinispanUserCacheProviderFactory.USER_CLEAR_CACHE_EVENTS, new ClearCacheEvent(), true);
     }
 
     public UserProvider getDelegate() {
@@ -97,10 +105,8 @@ public class UserCacheSession implements UserCache {
     }
 
     public void registerUserInvalidation(RealmModel realm,CachedUser user) {
-        invalidations.add(user.getId());
+        cache.userUpdatedInvalidations(user.getId(), user.getUsername(), user.getEmail(), user.getRealm(), invalidations);
-        if (user.getEmail() != null) invalidations.add(getUserByEmailCacheKey(realm.getId(), user.getEmail()));
+        invalidationEvents.add(UserUpdatedEvent.create(user.getId(), user.getUsername(), user.getEmail(), user.getRealm()));
-        invalidations.add(getUserByUsernameCacheKey(realm.getId(), user.getUsername()));
-        if (realm.isIdentityFederationEnabled()) invalidations.add(getFederatedIdentityLinksCacheKey(user.getId()));
     }
 
     @Override
@@ -108,10 +114,8 @@ public class UserCacheSession implements UserCache {
         if (user instanceof CachedUserModel) {
             ((CachedUserModel)user).invalidate();
         } else {
-            invalidations.add(user.getId());
+            cache.userUpdatedInvalidations(user.getId(), user.getUsername(), user.getEmail(), realm.getId(), invalidations);
-            if (user.getEmail() != null) invalidations.add(getUserByEmailCacheKey(realm.getId(), user.getEmail()));
+            invalidationEvents.add(UserUpdatedEvent.create(user.getId(), user.getUsername(), user.getEmail(), realm.getId()));
-            invalidations.add(getUserByUsernameCacheKey(realm.getId(), user.getUsername()));
-            if (realm.isIdentityFederationEnabled()) invalidations.add(getFederatedIdentityLinksCacheKey(user.getId()));
         }
     }
 
@@ -127,6 +131,8 @@ public class UserCacheSession implements UserCache {
         for (String invalidation : invalidations) {
             cache.invalidateObject(invalidation);
         }
+
+        cache.sendInvalidationEvents(session, invalidationEvents);
     }
 
     private KeycloakTransaction getTransaction() {
@@ -201,19 +207,23 @@ public class UserCacheSession implements UserCache {
         return adapter;
     }
 
-    public String getUserByUsernameCacheKey(String realmId, String username) {
+    static String getUserByUsernameCacheKey(String realmId, String username) {
         return realmId + ".username." + username;
     }
 
-    public String getUserByEmailCacheKey(String realmId, String email) {
+    static String getUserByEmailCacheKey(String realmId, String email) {
         return realmId + ".email." + email;
     }
 
-    public String getUserByFederatedIdentityCacheKey(String realmId, FederatedIdentityModel socialLink) {
+    private static String getUserByFederatedIdentityCacheKey(String realmId, FederatedIdentityModel socialLink) {
-        return realmId + ".idp." + socialLink.getIdentityProvider() + "." + socialLink.getUserId();
+        return getUserByFederatedIdentityCacheKey(realmId, socialLink.getIdentityProvider(), socialLink.getUserId());
     }
 
-    public String getFederatedIdentityLinksCacheKey(String userId) {
+    static String getUserByFederatedIdentityCacheKey(String realmId, String identityProvider, String socialUserId) {
+        return realmId + ".idp." + identityProvider + "." + socialUserId;
+    }
+
+    static String getFederatedIdentityLinksCacheKey(String userId) {
         return userId + ".idplinks";
     }
 
@@ -655,27 +665,32 @@ public class UserCacheSession implements UserCache {
 
     @Override
     public void updateConsent(RealmModel realm, String userId, UserConsentModel consent) {
-        invalidations.add(getConsentCacheKey(userId));
+        invalidateConsent(userId);
         getDelegate().updateConsent(realm, userId, consent);
     }
 
     @Override
     public boolean revokeConsentForClient(RealmModel realm, String userId, String clientInternalId) {
-        invalidations.add(getConsentCacheKey(userId));
+        invalidateConsent(userId);
         return getDelegate().revokeConsentForClient(realm, userId, clientInternalId);
     }
 
-    public String getConsentCacheKey(String userId) {
+    static String getConsentCacheKey(String userId) {
         return userId + ".consents";
     }
 
 
     @Override
     public void addConsent(RealmModel realm, String userId, UserConsentModel consent) {
-        invalidations.add(getConsentCacheKey(userId));
+        invalidateConsent(userId);
         getDelegate().addConsent(realm, userId, consent);
     }
 
+    private void invalidateConsent(String userId) {
+        cache.consentInvalidation(userId, invalidations);
+        invalidationEvents.add(UserConsentsUpdatedEvent.create(userId));
+    }
+
     @Override
     public UserConsentModel getConsentByClient(RealmModel realm, String userId, String clientId) {
         logger.tracev("getConsentByClient: {0}", userId);
@@ -754,7 +769,7 @@ public class UserCacheSession implements UserCache {
     public UserModel addUser(RealmModel realm, String id, String username, boolean addDefaultRoles, boolean addDefaultRequiredActions) {
         UserModel user = getDelegate().addUser(realm, id, username, addDefaultRoles, addDefaultRoles);
         // just in case the transaction is rolled back you need to invalidate the user and all cache queries for that user
-        invalidateUser(realm, user);
+        fullyInvalidateUser(realm, user);
         managedUsers.put(user.getId(), user);
         return user;
     }
@@ -763,94 +778,89 @@ public class UserCacheSession implements UserCache {
     public UserModel addUser(RealmModel realm, String username) {
         UserModel user = getDelegate().addUser(realm, username);
         // just in case the transaction is rolled back you need to invalidate the user and all cache queries for that user
-        invalidateUser(realm, user);
+        fullyInvalidateUser(realm, user);
         managedUsers.put(user.getId(), user);
         return user;
     }
 
-    protected void invalidateUser(RealmModel realm, UserModel user) {
+    // just in case the transaction is rolled back you need to invalidate the user and all cache queries for that user
-        // just in case the transaction is rolled back you need to invalidate the user and all cache queries for that user
+    protected void fullyInvalidateUser(RealmModel realm, UserModel user) {
+        Set<FederatedIdentityModel> federatedIdentities = realm.isIdentityFederationEnabled() ? getFederatedIdentities(user, realm) : null;
 
-        if (realm.isIdentityFederationEnabled()) {
+        UserFullInvalidationEvent event = UserFullInvalidationEvent.create(user.getId(), user.getUsername(), user.getEmail(), realm.getId(), realm.isIdentityFederationEnabled(), federatedIdentities);
-            // Invalidate all keys for lookup this user by any identityProvider link
-            Set<FederatedIdentityModel> federatedIdentities = getFederatedIdentities(user, realm);
-            for (FederatedIdentityModel socialLink : federatedIdentities) {
-                String fedIdentityCacheKey = getUserByFederatedIdentityCacheKey(realm.getId(), socialLink);
-                invalidations.add(fedIdentityCacheKey);
-            }
 
-            // Invalidate federationLinks of user
+        cache.fullUserInvalidation(user.getId(), user.getUsername(), user.getEmail(), realm.getId(), realm.isIdentityFederationEnabled(), event.getFederatedIdentities(), invalidations);
-            invalidations.add(getFederatedIdentityLinksCacheKey(user.getId()));
+        invalidationEvents.add(event);
-        }
-
-        invalidations.add(user.getId());
-        if (user.getEmail() != null) invalidations.add(getUserByEmailCacheKey(realm.getId(), user.getEmail()));
-        invalidations.add(getUserByUsernameCacheKey(realm.getId(), user.getUsername()));
     }
 
     @Override
     public boolean removeUser(RealmModel realm, UserModel user) {
-        invalidateUser(realm, user);
+        fullyInvalidateUser(realm, user);
         return getDelegate().removeUser(realm, user);
     }
 
     @Override
     public void addFederatedIdentity(RealmModel realm, UserModel user, FederatedIdentityModel socialLink) {
-        invalidations.add(getFederatedIdentityLinksCacheKey(user.getId()));
+        invalidateFederationLink(user.getId());
         getDelegate().addFederatedIdentity(realm, user, socialLink);
     }
 
     @Override
     public void updateFederatedIdentity(RealmModel realm, UserModel federatedUser, FederatedIdentityModel federatedIdentityModel) {
-        invalidations.add(getFederatedIdentityLinksCacheKey(federatedUser.getId()));
+        invalidateFederationLink(federatedUser.getId());
         getDelegate().updateFederatedIdentity(realm, federatedUser, federatedIdentityModel);
     }
 
+    private void invalidateFederationLink(String userId) {
+        cache.federatedIdentityLinkUpdatedInvalidation(userId, invalidations);
+        invalidationEvents.add(UserFederationLinkUpdatedEvent.create(userId));
+    }
+
     @Override
     public boolean removeFederatedIdentity(RealmModel realm, UserModel user, String socialProvider) {
         // Needs to invalidate both directions
         FederatedIdentityModel socialLink = getFederatedIdentity(user, socialProvider, realm);
-        invalidations.add(getFederatedIdentityLinksCacheKey(user.getId()));
+
-        if (socialLink != null) {
+        UserFederationLinkRemovedEvent event = UserFederationLinkRemovedEvent.create(user.getId(), realm.getId(), socialLink);
-            invalidations.add(getUserByFederatedIdentityCacheKey(realm.getId(), socialLink));
+        cache.federatedIdentityLinkRemovedInvalidation(user.getId(), realm.getId(), event.getIdentityProviderId(), event.getSocialUserId(), invalidations);
-        }
+        invalidationEvents.add(event);
 
         return getDelegate().removeFederatedIdentity(realm, user, socialProvider);
     }
 
     @Override
     public void grantToAllUsers(RealmModel realm, RoleModel role) {
-        realmInvalidations.add(realm.getId()); // easier to just invalidate whole realm
+        addRealmInvalidation(realm.getId()); // easier to just invalidate whole realm
         getDelegate().grantToAllUsers(realm, role);
     }
 
     @Override
     public void preRemove(RealmModel realm) {
-        realmInvalidations.add(realm.getId());
+        addRealmInvalidation(realm.getId());
         getDelegate().preRemove(realm);
     }
 
     @Override
     public void preRemove(RealmModel realm, RoleModel role) {
-        realmInvalidations.add(realm.getId()); // easier to just invalidate whole realm
+        addRealmInvalidation(realm.getId()); // easier to just invalidate whole realm
         getDelegate().preRemove(realm, role);
     }
     @Override
     public void preRemove(RealmModel realm, GroupModel group) {
-        realmInvalidations.add(realm.getId()); // easier to just invalidate whole realm
+        addRealmInvalidation(realm.getId()); // easier to just invalidate whole realm
         getDelegate().preRemove(realm, group);
     }
 
 
     @Override
     public void preRemove(RealmModel realm, UserFederationProviderModel link) {
-        realmInvalidations.add(realm.getId()); // easier to just invalidate whole realm
+        addRealmInvalidation(realm.getId()); // easier to just invalidate whole realm
         getDelegate().preRemove(realm, link);
     }
 
     @Override
     public void preRemove(RealmModel realm, ClientModel client) {
-        realmInvalidations.add(realm.getId()); // easier to just invalidate whole realm
+        addRealmInvalidation(realm.getId()); // easier to just invalidate whole realm
         getDelegate().preRemove(realm, client);
     }
 
@@ -862,9 +872,14 @@ public class UserCacheSession implements UserCache {
     @Override
     public void preRemove(RealmModel realm, ComponentModel component) {
         if (!component.getProviderType().equals(UserStorageProvider.class.getName())) return;
-        realmInvalidations.add(realm.getId()); // easier to just invalidate whole realm
+        addRealmInvalidation(realm.getId()); // easier to just invalidate whole realm
         getDelegate().preRemove(realm, component);
 
     }
 
+    private void addRealmInvalidation(String realmId) {
+        realmInvalidations.add(realmId);
+        invalidationEvents.add(UserCacheRealmInvalidationEvent.create(realmId));
+    }
+
 }

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/entities/CachedRealm.java

@@ -135,7 +135,6 @@ public class CachedRealm extends AbstractExtendableRevisioned {
     }
 
     protected List<String> defaultGroups = new LinkedList<String>();
-    protected Set<String> groups = new HashSet<String>();
     protected List<String> clientTemplates= new LinkedList<>();
     protected boolean internationalizationEnabled;
     protected Set<String> supportedLocales;
@@ -237,9 +236,7 @@ public class CachedRealm extends AbstractExtendableRevisioned {
                 executionsById.put(execution.getId(), execution);
             }
         }
-        for (GroupModel group : model.getGroups()) {
+
-            groups.add(group.getId());
-        }
         for (AuthenticatorConfigModel authenticator : model.getAuthenticatorConfigs()) {
             authenticatorConfigs.put(authenticator.getId(), authenticator);
         }
@@ -541,10 +538,6 @@ public class CachedRealm extends AbstractExtendableRevisioned {
         return clientAuthenticationFlow;
     }
 
-    public Set<String> getGroups() {
-        return groups;
-    }
-
     public List<String> getDefaultGroups() {
         return defaultGroups;
     }

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/entities/ClientTemplateQuery.java

@@ -1,11 +0,0 @@
-package org.keycloak.models.cache.infinispan.entities;
-
-import java.util.Set;
-
-/**
- * @author <a href="mailto:bill@burkecentral.com">Bill Burke</a>
- * @version $Revision: 1 $
- */
-public interface ClientTemplateQuery extends InRealm {
-    Set<String> getTemplates();
-}

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/entities/GroupListQuery.java

@@ -1,8 +1,6 @@
-package org.keycloak.models.cache.infinispan;
+package org.keycloak.models.cache.infinispan.entities;
 
 import org.keycloak.models.RealmModel;
-import org.keycloak.models.cache.infinispan.entities.AbstractRevisioned;
-import org.keycloak.models.cache.infinispan.entities.GroupQuery;
 
 import java.util.Set;
 

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/entities/RoleListQuery.java

@@ -59,7 +59,8 @@ public class RoleListQuery extends AbstractRevisioned implements RoleQuery, InCl
     public String toString() {
         return "RoleListQuery{" +
                 "id='" + getId() + "'" +
-                "realmName='" + realmName + '\'' +
+                ", realmName='" + realmName + '\'' +
+                ", clientUuid='" + client + '\'' +
                 '}';
     }
 }

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/events/ClientAddedEvent.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.models.cache.infinispan.events;
+
+import java.util.Set;
+
+import org.keycloak.models.cache.infinispan.RealmCacheManager;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class ClientAddedEvent extends InvalidationEvent implements RealmCacheInvalidationEvent {
+
+    private String clientUuid;
+    private String clientId;
+    private String realmId;
+
+    public static ClientAddedEvent create(String clientUuid, String clientId, String realmId) {
+        ClientAddedEvent event = new ClientAddedEvent();
+        event.clientUuid = clientUuid;
+        event.clientId = clientId;
+        event.realmId = realmId;
+        return event;
+    }
+
+    @Override
+    public String getId() {
+        return clientUuid;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("ClientAddedEvent [ realmId=%s, clientUuid=%s, clientId=%s ]", realmId, clientUuid, clientId);
+    }
+
+    @Override
+    public void addInvalidations(RealmCacheManager realmCache, Set<String> invalidations) {
+        realmCache.clientAdded(realmId, clientUuid, clientId, invalidations);
+    }
+}

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/events/ClientRemovedEvent.java

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.models.cache.infinispan.events;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.keycloak.models.ClientModel;
+import org.keycloak.models.RoleModel;
+import org.keycloak.models.cache.infinispan.RealmCacheManager;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class ClientRemovedEvent extends InvalidationEvent implements RealmCacheInvalidationEvent {
+
+    private String clientUuid;
+    private String clientId;
+    private String realmId;
+    // roleId -> roleName
+    private Map<String, String> clientRoles;
+
+    public static ClientRemovedEvent create(ClientModel client) {
+        ClientRemovedEvent event = new ClientRemovedEvent();
+
+        event.realmId = client.getRealm().getId();
+        event.clientUuid = client.getId();
+        event.clientId = client.getClientId();
+        event.clientRoles = new HashMap<>();
+        for (RoleModel clientRole : client.getRoles()) {
+            event.clientRoles.put(clientRole.getId(), clientRole.getName());
+        }
+
+        return event;
+    }
+
+    @Override
+    public String getId() {
+        return clientUuid;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("ClientRemovedEvent [ realmId=%s, clientUuid=%s, clientId=%s, clientRoleIds=%s ]", realmId, clientUuid, clientId, clientRoles);
+    }
+
+    @Override
+    public void addInvalidations(RealmCacheManager realmCache, Set<String> invalidations) {
+        realmCache.clientRemoval(realmId, clientUuid, clientId, invalidations);
+
+        // Separate iteration for all client roles to invalidate records dependent on them
+        for (Map.Entry<String, String> clientRole : clientRoles.entrySet()) {
+            String roleId = clientRole.getKey();
+            String roleName = clientRole.getValue();
+            realmCache.roleRemoval(roleId, roleName, clientUuid, invalidations);
+        }
+    }
+}

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/events/ClientTemplateEvent.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.models.cache.infinispan.events;
+
+import java.util.Set;
+
+import org.keycloak.models.cache.infinispan.RealmCacheManager;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class ClientTemplateEvent extends InvalidationEvent implements RealmCacheInvalidationEvent {
+
+    private String clientTemplateId;
+
+    public static ClientTemplateEvent create(String clientTemplateId) {
+        ClientTemplateEvent event = new ClientTemplateEvent();
+        event.clientTemplateId = clientTemplateId;
+        return event;
+    }
+
+    @Override
+    public String getId() {
+        return clientTemplateId;
+    }
+
+
+    @Override
+    public String toString() {
+        return "ClientTemplateEvent [ " + clientTemplateId + " ]";
+    }
+
+    @Override
+    public void addInvalidations(RealmCacheManager realmCache, Set<String> invalidations) {
+        // Nothing. ID was already invalidated
+    }
+}

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/events/ClientUpdatedEvent.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.models.cache.infinispan.events;
+
+import java.util.Set;
+
+import org.keycloak.models.cache.infinispan.RealmCacheManager;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class ClientUpdatedEvent extends InvalidationEvent implements RealmCacheInvalidationEvent {
+
+    private String clientUuid;
+    private String clientId;
+    private String realmId;
+
+    public static ClientUpdatedEvent create(String clientUuid, String clientId, String realmId) {
+        ClientUpdatedEvent event = new ClientUpdatedEvent();
+        event.clientUuid = clientUuid;
+        event.clientId = clientId;
+        event.realmId = realmId;
+        return event;
+    }
+
+    @Override
+    public String getId() {
+        return clientUuid;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("ClientUpdatedEvent [ realmId=%s, clientUuid=%s, clientId=%s ]", realmId, clientUuid, clientId);
+    }
+
+    @Override
+    public void addInvalidations(RealmCacheManager realmCache, Set<String> invalidations) {
+        realmCache.clientUpdated(realmId, clientUuid, clientId, invalidations);
+    }
+}

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/events/GroupAddedEvent.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.models.cache.infinispan.events;
+
+import java.util.Set;
+
+import org.keycloak.models.cache.infinispan.RealmCacheManager;
+
+/**
+ *
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class GroupAddedEvent extends InvalidationEvent implements RealmCacheInvalidationEvent {
+
+    private String groupId;
+    private String realmId;
+
+    public static GroupAddedEvent create(String groupId, String realmId) {
+        GroupAddedEvent event = new GroupAddedEvent();
+        event.realmId = realmId;
+        event.groupId = groupId;
+        return event;
+    }
+
+    @Override
+    public String getId() {
+        return groupId;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("GroupAddedEvent [ realmId=%s, groupId=%s ]", realmId, groupId);
+    }
+
+    @Override
+    public void addInvalidations(RealmCacheManager realmCache, Set<String> invalidations) {
+        realmCache.groupQueriesInvalidations(realmId, invalidations);
+    }
+}

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/events/GroupMovedEvent.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.models.cache.infinispan.events;
+
+import java.util.Set;
+
+import org.keycloak.models.GroupModel;
+import org.keycloak.models.cache.infinispan.RealmCacheManager;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class GroupMovedEvent extends InvalidationEvent implements RealmCacheInvalidationEvent {
+
+    private String groupId;
+    private String newParentId; // null if moving to top-level
+    private String oldParentId; // null if moving from top-level
+    private String realmId;
+
+    public static GroupMovedEvent create(GroupModel group, GroupModel toParent, String realmId) {
+        GroupMovedEvent event = new GroupMovedEvent();
+        event.realmId = realmId;
+        event.groupId = group.getId();
+        event.oldParentId = group.getParentId();
+        event.newParentId = toParent==null ? null : toParent.getId();
+        return event;
+    }
+
+    @Override
+    public String getId() {
+        return groupId;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("GroupMovedEvent [ realmId=%s, groupId=%s, newParentId=%s, oldParentId=%s ]", realmId, groupId, newParentId, oldParentId);
+    }
+
+    @Override
+    public void addInvalidations(RealmCacheManager realmCache, Set<String> invalidations) {
+        realmCache.groupQueriesInvalidations(realmId, invalidations);
+        if (newParentId != null) {
+            invalidations.add(newParentId);
+        }
+        if (oldParentId != null) {
+            invalidations.add(oldParentId);
+        }
+    }
+}

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/events/GroupRemovedEvent.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.models.cache.infinispan.events;
+
+import java.util.Set;
+
+import org.keycloak.models.GroupModel;
+import org.keycloak.models.cache.infinispan.RealmCacheManager;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class GroupRemovedEvent extends InvalidationEvent implements RealmCacheInvalidationEvent {
+
+    private String groupId;
+    private String parentId;
+    private String realmId;
+
+    public static GroupRemovedEvent create(GroupModel group, String realmId) {
+        GroupRemovedEvent event = new GroupRemovedEvent();
+        event.realmId = realmId;
+        event.groupId = group.getId();
+        event.parentId = group.getParentId();
+        return event;
+    }
+
+    @Override
+    public String getId() {
+        return groupId;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("GroupRemovedEvent [ realmId=%s, groupId=%s, parentId=%s ]", realmId, groupId, parentId);
+    }
+
+    @Override
+    public void addInvalidations(RealmCacheManager realmCache, Set<String> invalidations) {
+        realmCache.groupQueriesInvalidations(realmId, invalidations);
+        if (parentId != null) {
+            invalidations.add(parentId);
+        }
+    }
+}

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/events/GroupUpdatedEvent.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.models.cache.infinispan.events;
+
+import java.util.Set;
+
+import org.keycloak.models.cache.infinispan.RealmCacheManager;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class GroupUpdatedEvent extends InvalidationEvent implements RealmCacheInvalidationEvent {
+
+    private String groupId;
+
+    public static GroupUpdatedEvent create(String groupId) {
+        GroupUpdatedEvent event = new GroupUpdatedEvent();
+        event.groupId = groupId;
+        return event;
+    }
+
+    @Override
+    public String getId() {
+        return groupId;
+    }
+
+
+    @Override
+    public String toString() {
+        return "GroupUpdatedEvent [ " + groupId + " ]";
+    }
+
+    @Override
+    public void addInvalidations(RealmCacheManager realmCache, Set<String> invalidations) {
+        // Nothing. ID already invalidated
+    }
+}

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/events/InvalidationEvent.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.models.cache.infinispan.events;
+
+import org.keycloak.cluster.ClusterEvent;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public abstract class InvalidationEvent implements ClusterEvent {
+
+    public abstract String getId();
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode() * 13 + getId().hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) return false;
+        if (!obj.getClass().equals(this.getClass())) return false;
+
+        InvalidationEvent that = (InvalidationEvent) obj;
+        if (!that.getId().equals(getId())) return false;
+        return true;
+    }
+}

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/events/RealmCacheInvalidationEvent.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.models.cache.infinispan.events;
+
+import java.util.Set;
+
+import org.keycloak.models.cache.infinispan.RealmCacheManager;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public interface RealmCacheInvalidationEvent {
+
+    void addInvalidations(RealmCacheManager realmCache, Set<String> invalidations);
+
+}

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/events/RealmRemovedEvent.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.models.cache.infinispan.events;
+
+import java.util.Set;
+
+import org.keycloak.models.cache.infinispan.RealmCacheManager;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class RealmRemovedEvent extends InvalidationEvent implements RealmCacheInvalidationEvent {
+
+    private String realmId;
+    private String realmName;
+
+    public static RealmRemovedEvent create(String realmId, String realmName) {
+        RealmRemovedEvent event = new RealmRemovedEvent();
+        event.realmId = realmId;
+        event.realmName = realmName;
+        return event;
+    }
+
+    @Override
+    public String getId() {
+        return realmId;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("RealmRemovedEvent [ realmId=%s, realmName=%s ]", realmId, realmName);
+    }
+
+    @Override
+    public void addInvalidations(RealmCacheManager realmCache, Set<String> invalidations) {
+        realmCache.realmRemoval(realmId, realmName, invalidations);
+    }
+}

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/events/RealmUpdatedEvent.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.models.cache.infinispan.events;
+
+import java.util.Set;
+
+import org.keycloak.models.cache.infinispan.RealmCacheManager;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class RealmUpdatedEvent extends InvalidationEvent implements RealmCacheInvalidationEvent {
+
+    private String realmId;
+    private String realmName;
+
+    public static RealmUpdatedEvent create(String realmId, String realmName) {
+        RealmUpdatedEvent event = new RealmUpdatedEvent();
+        event.realmId = realmId;
+        event.realmName = realmName;
+        return event;
+    }
+
+    @Override
+    public String getId() {
+        return realmId;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("RealmUpdatedEvent [ realmId=%s, realmName=%s ]", realmId, realmName);
+    }
+
+    @Override
+    public void addInvalidations(RealmCacheManager realmCache, Set<String> invalidations) {
+        realmCache.realmUpdated(realmId, realmName, invalidations);
+    }
+}

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/events/RoleAddedEvent.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.models.cache.infinispan.events;
+
+import java.util.Set;
+
+import org.keycloak.models.cache.infinispan.RealmCacheManager;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class RoleAddedEvent extends InvalidationEvent implements RealmCacheInvalidationEvent {
+
+    private String roleId;
+    private String containerId;
+
+    public static RoleAddedEvent create(String roleId, String containerId) {
+        RoleAddedEvent event = new RoleAddedEvent();
+        event.roleId = roleId;
+        event.containerId = containerId;
+        return event;
+    }
+
+    @Override
+    public String getId() {
+        return roleId;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("RoleAddedEvent [ roleId=%s, containerId=%s ]", roleId, containerId);
+    }
+
+    @Override
+    public void addInvalidations(RealmCacheManager realmCache, Set<String> invalidations) {
+        realmCache.roleAdded(containerId, invalidations);
+    }
+}

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/events/RoleRemovedEvent.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.models.cache.infinispan.events;
+
+import java.util.Set;
+
+import org.keycloak.models.cache.infinispan.RealmCacheManager;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class RoleRemovedEvent extends InvalidationEvent implements RealmCacheInvalidationEvent {
+
+    private String roleId;
+    private String roleName;
+    private String containerId;
+
+    public static RoleRemovedEvent create(String roleId, String roleName, String containerId) {
+        RoleRemovedEvent event = new RoleRemovedEvent();
+        event.roleId = roleId;
+        event.roleName = roleName;
+        event.containerId = containerId;
+        return event;
+    }
+
+    @Override
+    public String getId() {
+        return roleId;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("RoleRemovedEvent [ roleId=%s, containerId=%s ]", roleId, containerId);
+    }
+
+    @Override
+    public void addInvalidations(RealmCacheManager realmCache, Set<String> invalidations) {
+        realmCache.roleRemoval(roleId, roleName, containerId, invalidations);
+    }
+}

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/events/RoleUpdatedEvent.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.models.cache.infinispan.events;
+
+import java.util.Set;
+
+import org.keycloak.models.cache.infinispan.RealmCacheManager;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class RoleUpdatedEvent extends InvalidationEvent implements RealmCacheInvalidationEvent {
+
+    private String roleId;
+    private String roleName;
+    private String containerId;
+
+    public static RoleUpdatedEvent create(String roleId, String roleName, String containerId) {
+        RoleUpdatedEvent event = new RoleUpdatedEvent();
+        event.roleId = roleId;
+        event.roleName = roleName;
+        event.containerId = containerId;
+        return event;
+    }
+
+    @Override
+    public String getId() {
+        return roleId;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("RoleUpdatedEvent [ roleId=%s, roleName=%s, containerId=%s ]", roleId, roleName, containerId);
+    }
+
+    @Override
+    public void addInvalidations(RealmCacheManager realmCache, Set<String> invalidations) {
+        realmCache.roleUpdated(containerId, roleName, invalidations);
+    }
+}

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/events/UserCacheInvalidationEvent.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.models.cache.infinispan.events;
+
+import java.util.Set;
+
+import org.keycloak.models.cache.infinispan.UserCacheManager;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public interface UserCacheInvalidationEvent {
+
+    void addInvalidations(UserCacheManager userCache, Set<String> invalidations);
+
+}

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/events/UserCacheRealmInvalidationEvent.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.models.cache.infinispan.events;
+
+import java.util.Set;
+
+import org.keycloak.models.cache.infinispan.UserCacheManager;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class UserCacheRealmInvalidationEvent  extends InvalidationEvent implements UserCacheInvalidationEvent {
+
+    private String realmId;
+
+    public static UserCacheRealmInvalidationEvent create(String realmId) {
+        UserCacheRealmInvalidationEvent event = new UserCacheRealmInvalidationEvent();
+        event.realmId = realmId;
+        return event;
+    }
+
+    @Override
+    public String getId() {
+        return realmId; // Just a placeholder
+    }
+
+    @Override
+    public String toString() {
+        return String.format("UserCacheRealmInvalidationEvent [ realmId=%s ]", realmId);
+    }
+
+    @Override
+    public void addInvalidations(UserCacheManager userCache, Set<String> invalidations) {
+        userCache.invalidateRealmUsers(realmId, invalidations);
+    }
+}

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/events/UserConsentsUpdatedEvent.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.models.cache.infinispan.events;
+
+import java.util.Set;
+
+import org.keycloak.models.cache.infinispan.UserCacheManager;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class UserConsentsUpdatedEvent extends InvalidationEvent implements UserCacheInvalidationEvent {
+
+    private String userId;
+
+    public static UserConsentsUpdatedEvent create(String userId) {
+        UserConsentsUpdatedEvent event = new UserConsentsUpdatedEvent();
+        event.userId = userId;
+        return event;
+    }
+
+    @Override
+    public String getId() {
+        return userId;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("UserConsentsUpdatedEvent [ userId=%s ]", userId);
+    }
+
+    @Override
+    public void addInvalidations(UserCacheManager userCache, Set<String> invalidations) {
+        userCache.consentInvalidation(userId, invalidations);
+    }
+}

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/events/UserFederationLinkRemovedEvent.java

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.models.cache.infinispan.events;
+
+import java.util.Set;
+
+import org.keycloak.models.FederatedIdentityModel;
+import org.keycloak.models.cache.infinispan.UserCacheManager;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class UserFederationLinkRemovedEvent extends InvalidationEvent implements UserCacheInvalidationEvent {
+
+    private String userId;
+    private String realmId;
+    private String identityProviderId;
+    private String socialUserId;
+
+    public static UserFederationLinkRemovedEvent create(String userId, String realmId, FederatedIdentityModel socialLink) {
+        UserFederationLinkRemovedEvent event = new UserFederationLinkRemovedEvent();
+        event.userId = userId;
+        event.realmId = realmId;
+        if (socialLink != null) {
+            event.identityProviderId = socialLink.getIdentityProvider();
+            event.socialUserId = socialLink.getUserId();
+        }
+        return event;
+    }
+
+    @Override
+    public String getId() {
+        return userId;
+    }
+
+    public String getRealmId() {
+        return realmId;
+    }
+
+    public String getIdentityProviderId() {
+        return identityProviderId;
+    }
+
+    public String getSocialUserId() {
+        return socialUserId;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("UserFederationLinkRemovedEvent [ userId=%s, identityProviderId=%s, socialUserId=%s ]", userId, identityProviderId, socialUserId);
+    }
+
+    @Override
+    public void addInvalidations(UserCacheManager userCache, Set<String> invalidations) {
+        userCache.federatedIdentityLinkRemovedInvalidation(userId, realmId, identityProviderId, socialUserId, invalidations);
+    }
+}

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/events/UserFederationLinkUpdatedEvent.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.models.cache.infinispan.events;
+
+import java.util.Set;
+
+import org.keycloak.models.cache.infinispan.UserCacheManager;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class UserFederationLinkUpdatedEvent extends InvalidationEvent implements UserCacheInvalidationEvent {
+
+    private String userId;
+
+    public static UserFederationLinkUpdatedEvent create(String userId) {
+        UserFederationLinkUpdatedEvent event = new UserFederationLinkUpdatedEvent();
+        event.userId = userId;
+        return event;
+    }
+
+    @Override
+    public String getId() {
+        return userId;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("UserFederationLinkUpdatedEvent [ userId=%s ]", userId);
+    }
+
+    @Override
+    public void addInvalidations(UserCacheManager userCache, Set<String> invalidations) {
+        userCache.federatedIdentityLinkUpdatedInvalidation(userId, invalidations);
+    }
+}

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/events/UserFullInvalidationEvent.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.models.cache.infinispan.events;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.keycloak.models.FederatedIdentityModel;
+import org.keycloak.models.cache.infinispan.UserCacheManager;
+
+/**
+ * Used when user added/removed
+ *
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class UserFullInvalidationEvent extends InvalidationEvent implements UserCacheInvalidationEvent {
+
+    private String userId;
+    private String username;
+    private String email;
+    private String realmId;
+    private boolean identityFederationEnabled;
+    private Map<String, String> federatedIdentities;
+
+    public static UserFullInvalidationEvent create(String userId, String username, String email, String realmId, boolean identityFederationEnabled, Collection<FederatedIdentityModel> federatedIdentities) {
+        UserFullInvalidationEvent event = new UserFullInvalidationEvent();
+        event.userId = userId;
+        event.username = username;
+        event.email = email;
+        event.realmId = realmId;
+
+        event.identityFederationEnabled = identityFederationEnabled;
+        if (identityFederationEnabled) {
+            event.federatedIdentities = new HashMap<>();
+            for (FederatedIdentityModel socialLink : federatedIdentities) {
+                event.federatedIdentities.put(socialLink.getIdentityProvider(), socialLink.getUserId());
+            }
+        }
+
+        return event;
+    }
+
+    @Override
+    public String getId() {
+        return userId;
+    }
+
+    public Map<String, String> getFederatedIdentities() {
+        return federatedIdentities;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("UserFullInvalidationEvent [ userId=%s, username=%s, email=%s ]", userId, username, email);
+    }
+
+    @Override
+    public void addInvalidations(UserCacheManager userCache, Set<String> invalidations) {
+        userCache.fullUserInvalidation(userId, username, email, realmId, identityFederationEnabled, federatedIdentities, invalidations);
+    }
+}

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/events/UserUpdatedEvent.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.models.cache.infinispan.events;
+
+import java.util.Set;
+
+import org.keycloak.models.cache.infinispan.UserCacheManager;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class UserUpdatedEvent extends InvalidationEvent implements UserCacheInvalidationEvent {
+
+    private String userId;
+    private String username;
+    private String email;
+    private String realmId;
+
+    public static UserUpdatedEvent create(String userId, String username, String email, String realmId) {
+        UserUpdatedEvent event = new UserUpdatedEvent();
+        event.userId = userId;
+        event.username = username;
+        event.email = email;
+        event.realmId = realmId;
+        return event;
+    }
+
+    @Override
+    public String getId() {
+        return userId;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("UserUpdatedEvent [ userId=%s, username=%s, email=%s ]", userId, username, email);
+    }
+
+    @Override
+    public void addInvalidations(UserCacheManager userCache, Set<String> invalidations) {
+        userCache.userUpdatedInvalidations(userId, username, email, realmId, invalidations);
+    }
+}

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/stream/ClientQueryPredicate.java

@@ -1,48 +0,0 @@
-package org.keycloak.models.cache.infinispan.stream;
-
-import org.jboss.logging.Logger;
-import org.keycloak.models.cache.infinispan.entities.ClientQuery;
-import org.keycloak.models.cache.infinispan.entities.Revisioned;
-
-import java.io.Serializable;
-import java.util.Map;
-import java.util.function.Predicate;
-
-/**
- * @author <a href="mailto:bill@burkecentral.com">Bill Burke</a>
- * @version $Revision: 1 $
- */
-public class ClientQueryPredicate implements Predicate<Map.Entry<String, Revisioned>>, Serializable {
-    protected static final Logger logger = Logger.getLogger(ClientQueryPredicate.class);
-    private String client;
-    private String inRealm;
-
-    public static ClientQueryPredicate create() {
-        return new ClientQueryPredicate();
-    }
-
-    public ClientQueryPredicate client(String client) {
-        this.client = client;
-        return this;
-    }
-
-    public ClientQueryPredicate inRealm(String inRealm) {
-        this.inRealm = inRealm;
-        return this;
-    }
-
-
-
-
-
-    @Override
-    public boolean test(Map.Entry<String, Revisioned> entry) {
-        Object value = entry.getValue();
-        if (value == null) return false;
-        if (!(value instanceof ClientQuery)) return false;
-        ClientQuery query = (ClientQuery)value;
-        if (client != null && !query.getClients().contains(client)) return false;
-        if (inRealm != null && !query.getRealm().equals(inRealm)) return false;
-        return true;
-    }
-}

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/stream/ClientTemplateQueryPredicate.java

@@ -1,40 +0,0 @@
-package org.keycloak.models.cache.infinispan.stream;
-
-import org.keycloak.models.cache.infinispan.entities.ClientTemplateQuery;
-import org.keycloak.models.cache.infinispan.entities.Revisioned;
-
-import java.io.Serializable;
-import java.util.Map;
-import java.util.function.Predicate;
-
-/**
- * @author <a href="mailto:bill@burkecentral.com">Bill Burke</a>
- * @version $Revision: 1 $
- */
-public class ClientTemplateQueryPredicate implements Predicate<Map.Entry<String, Revisioned>>, Serializable {
-    private String template;
-
-    public static ClientTemplateQueryPredicate create() {
-        return new ClientTemplateQueryPredicate();
-    }
-
-    public ClientTemplateQueryPredicate template(String template) {
-        this.template = template;
-        return this;
-    }
-
-
-
-
-
-    @Override
-    public boolean test(Map.Entry<String, Revisioned> entry) {
-        Object value = entry.getValue();
-        if (value == null) return false;
-        if (!(value instanceof ClientTemplateQuery)) return false;
-        ClientTemplateQuery query = (ClientTemplateQuery)value;
-
-
-        return query.getTemplates().contains(template);
-    }
-}

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/stream/GroupQueryPredicate.java

@@ -1,40 +0,0 @@
-package org.keycloak.models.cache.infinispan.stream;
-
-import org.keycloak.models.cache.infinispan.entities.GroupQuery;
-import org.keycloak.models.cache.infinispan.entities.Revisioned;
-
-import java.io.Serializable;
-import java.util.Map;
-import java.util.function.Predicate;
-
-/**
- * @author <a href="mailto:bill@burkecentral.com">Bill Burke</a>
- * @version $Revision: 1 $
- */
-public class GroupQueryPredicate implements Predicate<Map.Entry<String, Revisioned>>, Serializable {
-    private String group;
-
-    public static GroupQueryPredicate create() {
-        return new GroupQueryPredicate();
-    }
-
-    public GroupQueryPredicate group(String group) {
-        this.group = group;
-        return this;
-    }
-
-
-
-
-
-    @Override
-    public boolean test(Map.Entry<String, Revisioned> entry) {
-        Object value = entry.getValue();
-        if (value == null) return false;
-        if (!(value instanceof GroupQuery)) return false;
-        GroupQuery query = (GroupQuery)value;
-
-
-        return query.getGroups().contains(group);
-    }
-}

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/stream/RealmQueryPredicate.java

@@ -1,40 +0,0 @@
-package org.keycloak.models.cache.infinispan.stream;
-
-import org.keycloak.models.cache.infinispan.entities.RealmQuery;
-import org.keycloak.models.cache.infinispan.entities.Revisioned;
-
-import java.io.Serializable;
-import java.util.Map;
-import java.util.function.Predicate;
-
-/**
- * @author <a href="mailto:bill@burkecentral.com">Bill Burke</a>
- * @version $Revision: 1 $
- */
-public class RealmQueryPredicate implements Predicate<Map.Entry<String, Revisioned>>, Serializable {
-    private String realm;
-
-    public static RealmQueryPredicate create() {
-        return new RealmQueryPredicate();
-    }
-
-    public RealmQueryPredicate realm(String realm) {
-        this.realm = realm;
-        return this;
-    }
-
-
-
-
-
-    @Override
-    public boolean test(Map.Entry<String, Revisioned> entry) {
-        Object value = entry.getValue();
-        if (value == null) return false;
-        if (!(value instanceof RealmQuery)) return false;
-        RealmQuery query = (RealmQuery)value;
-
-
-        return query.getRealms().contains(realm);
-    }
-}

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/stream/RoleQueryPredicate.java

@@ -1,40 +0,0 @@
-package org.keycloak.models.cache.infinispan.stream;
-
-import org.keycloak.models.cache.infinispan.entities.Revisioned;
-import org.keycloak.models.cache.infinispan.entities.RoleQuery;
-
-import java.io.Serializable;
-import java.util.Map;
-import java.util.function.Predicate;
-
-/**
- * @author <a href="mailto:bill@burkecentral.com">Bill Burke</a>
- * @version $Revision: 1 $
- */
-public class RoleQueryPredicate implements Predicate<Map.Entry<String, Revisioned>>, Serializable {
-    private String role;
-
-    public static RoleQueryPredicate create() {
-        return new RoleQueryPredicate();
-    }
-
-    public RoleQueryPredicate role(String role) {
-        this.role = role;
-        return this;
-    }
-
-
-
-
-
-    @Override
-    public boolean test(Map.Entry<String, Revisioned> entry) {
-        Object value = entry.getValue();
-        if (value == null) return false;
-        if (!(value instanceof RoleQuery)) return false;
-        RoleQuery query = (RoleQuery)value;
-
-
-        return query.getRoles().contains(role);
-    }
-}

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/initializer/InfinispanUserSessionInitializer.java

@@ -92,13 +92,13 @@ public class InfinispanUserSessionInitializer {
 
 
     private boolean isFinished() {
-        InitializerState state = (InitializerState) workCache.get(stateKey);
+        InitializerState state = getStateFromCache();
         return state != null && state.isFinished();
     }
 
 
     private InitializerState getOrCreateInitializerState() {
-        InitializerState state = (InitializerState) workCache.get(stateKey);
+        InitializerState state = getStateFromCache();
         if (state == null) {
             final int[] count = new int[1];
 
@@ -128,6 +128,12 @@ public class InfinispanUserSessionInitializer {
 
     }
 
+    private InitializerState getStateFromCache() {
+        // TODO: We ignore cacheStore for now, so that in Cross-DC scenario (with RemoteStore enabled) is the remoteStore ignored. This means that every DC needs to load offline sessions separately.
+        return (InitializerState) workCache.getAdvancedCache()
+                .withFlags(Flag.SKIP_CACHE_STORE, Flag.SKIP_CACHE_LOAD)
+                .get(stateKey);
+    }
 
     private void saveStateToCache(final InitializerState state) {
 
@@ -138,8 +144,9 @@ public class InfinispanUserSessionInitializer {
             public void run() {
 
                 // Save this synchronously to ensure all nodes read correct state
+                // TODO: We ignore cacheStore for now, so that in Cross-DC scenario (with RemoteStore enabled) is the remoteStore ignored. This means that every DC needs to load offline sessions separately.
                 InfinispanUserSessionInitializer.this.workCache.getAdvancedCache().
-                        withFlags(Flag.IGNORE_RETURN_VALUES, Flag.FORCE_SYNCHRONOUS)
+                        withFlags(Flag.IGNORE_RETURN_VALUES, Flag.FORCE_SYNCHRONOUS, Flag.SKIP_CACHE_STORE, Flag.SKIP_CACHE_LOAD)
                         .put(stateKey, state);
             }
 

model/infinispan/src/test/java/org/keycloak/cluster/infinispan/ConcurrencyJDGRemoteCacheTest.java

@@ -0,0 +1,231 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.cluster.infinispan;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.infinispan.Cache;
+import org.infinispan.client.hotrod.Flag;
+import org.infinispan.client.hotrod.RemoteCache;
+import org.infinispan.client.hotrod.annotation.ClientCacheEntryCreated;
+import org.infinispan.client.hotrod.annotation.ClientCacheEntryModified;
+import org.infinispan.client.hotrod.annotation.ClientListener;
+import org.infinispan.client.hotrod.event.ClientCacheEntryCreatedEvent;
+import org.infinispan.client.hotrod.event.ClientCacheEntryModifiedEvent;
+import org.infinispan.configuration.cache.Configuration;
+import org.infinispan.configuration.cache.ConfigurationBuilder;
+import org.infinispan.configuration.global.GlobalConfigurationBuilder;
+import org.infinispan.manager.DefaultCacheManager;
+import org.infinispan.manager.EmbeddedCacheManager;
+import org.infinispan.persistence.manager.PersistenceManager;
+import org.infinispan.persistence.remote.RemoteStore;
+import org.infinispan.persistence.remote.configuration.ExhaustedAction;
+import org.infinispan.persistence.remote.configuration.RemoteStoreConfigurationBuilder;
+import org.junit.Ignore;
+import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
+
+/**
+ * Test concurrency for remoteStore (backed by HotRod RemoteCaches) against external JDG
+ *
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+@Ignore
+public class ConcurrencyJDGRemoteCacheTest {
+
+    private static Map<String, EntryInfo> state = new HashMap<>();
+
+    public static void main(String[] args) throws Exception {
+        // Init map somehow
+        for (int i=0 ; i<100 ; i++) {
+            String key = "key-" + i;
+            state.put(key, new EntryInfo());
+        }
+
+        // Create caches, listeners and finally worker threads
+        Worker worker1 = createWorker(1);
+        Worker worker2 = createWorker(2);
+
+        // Start and join workers
+        worker1.start();
+        worker2.start();
+
+        worker1.join();
+        worker2.join();
+
+        // Output
+        for (Map.Entry<String, EntryInfo> entry : state.entrySet()) {
+            System.out.println(entry.getKey() + ":::" + entry.getValue());
+            worker1.cache.remove(entry.getKey());
+        }
+
+        // Finish JVM
+        worker1.cache.getCacheManager().stop();
+        worker2.cache.getCacheManager().stop();
+    }
+
+    private static Worker createWorker(int threadId) {
+        EmbeddedCacheManager manager = createManager(threadId);
+        Cache<String, Integer> cache = manager.getCache(InfinispanConnectionProvider.WORK_CACHE_NAME);
+
+        System.out.println("Retrieved cache: " + threadId);
+
+        RemoteStore remoteStore = cache.getAdvancedCache().getComponentRegistry().getComponent(PersistenceManager.class).getStores(RemoteStore.class).iterator().next();
+        HotRodListener listener = new HotRodListener();
+        remoteStore.getRemoteCache().addClientListener(listener);
+
+        return new Worker(cache, threadId);
+    }
+
+    private static EmbeddedCacheManager createManager(int threadId) {
+        System.setProperty("java.net.preferIPv4Stack", "true");
+        System.setProperty("jgroups.tcp.port", "53715");
+        GlobalConfigurationBuilder gcb = new GlobalConfigurationBuilder();
+
+        boolean clustered = false;
+        boolean async = false;
+        boolean allowDuplicateJMXDomains = true;
+
+        if (clustered) {
+            gcb = gcb.clusteredDefault();
+            gcb.transport().clusterName("test-clustering");
+        }
+
+        gcb.globalJmxStatistics().allowDuplicateDomains(allowDuplicateJMXDomains);
+
+        EmbeddedCacheManager cacheManager = new DefaultCacheManager(gcb.build());
+
+        Configuration invalidationCacheConfiguration = getCacheBackedByRemoteStore(threadId);
+
+        cacheManager.defineConfiguration(InfinispanConnectionProvider.WORK_CACHE_NAME, invalidationCacheConfiguration);
+        return cacheManager;
+
+    }
+
+    private static Configuration getCacheBackedByRemoteStore(int threadId) {
+        ConfigurationBuilder cacheConfigBuilder = new ConfigurationBuilder();
+
+        // int port = threadId==1 ? 11222 : 11322;
+        int port = 11222;
+
+        return cacheConfigBuilder.persistence().addStore(RemoteStoreConfigurationBuilder.class)
+                .fetchPersistentState(false)
+                .ignoreModifications(false)
+                .purgeOnStartup(false)
+                .preload(false)
+                .shared(true)
+                .remoteCacheName(InfinispanConnectionProvider.WORK_CACHE_NAME)
+                .rawValues(true)
+                .forceReturnValues(false)
+                .addServer()
+                    .host("localhost")
+                    .port(port)
+                .connectionPool()
+                    .maxActive(20)
+                    .exhaustedAction(ExhaustedAction.CREATE_NEW)
+                .async()
+                .   enabled(false).build();
+    }
+
+
+    @ClientListener
+    public static class HotRodListener {
+
+        //private AtomicInteger listenerCount = new AtomicInteger(0);
+
+        @ClientCacheEntryCreated
+        public void created(ClientCacheEntryCreatedEvent event) {
+            String cacheKey = (String) event.getKey();
+            state.get(cacheKey).successfulListenerWrites.incrementAndGet();
+        }
+
+        @ClientCacheEntryModified
+        public void updated(ClientCacheEntryModifiedEvent event) {
+            String cacheKey = (String) event.getKey();
+            state.get(cacheKey).successfulListenerWrites.incrementAndGet();
+        }
+
+    }
+
+
+    private static class Worker extends Thread {
+
+        private final Cache<String, Integer> cache;
+
+        private final int myThreadId;
+
+        private Worker(Cache<String, Integer> cache, int myThreadId) {
+            this.cache = cache;
+            this.myThreadId = myThreadId;
+        }
+
+        @Override
+        public void run() {
+            for (Map.Entry<String, EntryInfo> entry : state.entrySet()) {
+                String cacheKey = entry.getKey();
+                EntryInfo wrapper = state.get(cacheKey);
+
+                int val = getClusterStartupTime(this.cache, cacheKey, wrapper);
+                if (myThreadId == 1) {
+                    wrapper.th1.set(val);
+                } else {
+                    wrapper.th2.set(val);
+                }
+
+            }
+
+            System.out.println("Worker finished: " + myThreadId);
+        }
+
+    }
+
+    public static int getClusterStartupTime(Cache<String, Integer> cache, String cacheKey, EntryInfo wrapper) {
+        int startupTime = new Random().nextInt(1024);
+
+        // Concurrency doesn't work correctly with this
+        //Integer existingClusterStartTime = (Integer) cache.putIfAbsent(cacheKey, startupTime);
+
+        // Concurrency works fine with this
+        RemoteCache remoteCache = cache.getAdvancedCache().getComponentRegistry().getComponent(PersistenceManager.class).getStores(RemoteStore.class).iterator().next().getRemoteCache();
+        Integer existingClusterStartTime = (Integer) remoteCache.withFlags(Flag.FORCE_RETURN_VALUE).putIfAbsent(cacheKey, startupTime);
+
+        if (existingClusterStartTime == null) {
+            wrapper.successfulInitializations.incrementAndGet();
+            return startupTime;
+        } else {
+            return existingClusterStartTime;
+        }
+    }
+
+    private static class EntryInfo {
+        AtomicInteger successfulInitializations = new AtomicInteger(0);
+        AtomicInteger successfulListenerWrites = new AtomicInteger(0);
+        AtomicInteger th1 = new AtomicInteger();
+        AtomicInteger th2 = new AtomicInteger();
+
+        @Override
+        public String toString() {
+            return String.format("Inits: %d, listeners: %d, th1: %d, th2: %d", successfulInitializations.get(), successfulListenerWrites.get(), th1.get(), th2.get());
+        }
+    }
+
+
+
+}

model/jpa/src/main/java/org/keycloak/models/jpa/JpaRealmProvider.java

@@ -146,7 +146,8 @@ public class JpaRealmProvider implements RealmProvider {
         query.setParameter("realm", realm.getId());
         List<String> clients = query.getResultList();
         for (String client : clients) {
-            session.realms().removeClient(client, adapter);
+            // No need to go through cache. Clients were already invalidated
+            removeClient(client, adapter);
         }
 
         for (ClientTemplateEntity a : new LinkedList<>(realm.getClientTemplates())) {
@@ -154,7 +155,8 @@ public class JpaRealmProvider implements RealmProvider {
         }
 
         for (RoleModel role : adapter.getRoles()) {
-            session.realms().removeRole(adapter, role);
+            // No need to go through cache. Roles were already invalidated
+            removeRole(adapter, role);
         }
 
 
@@ -486,7 +488,8 @@ public class JpaRealmProvider implements RealmProvider {
         session.users().preRemove(realm, client);
 
         for (RoleModel role : client.getRoles()) {
-            client.removeRole(role);
+            // No need to go through cache. Roles were already invalidated
+            removeRole(realm, role);
         }
 
         ClientEntity clientEntity = ((ClientAdapter)client).getEntity();

model/jpa/src/main/java/org/keycloak/models/jpa/RealmAdapter.java

@@ -952,13 +952,6 @@ public class RealmAdapter implements RealmModel, JpaModel<RealmEntity> {
         return session.realms().getRoleById(id, this);
     }
 
-    @Override
-    public boolean removeRoleById(String id) {
-        RoleModel role = getRoleById(id);
-        if (role == null) return false;
-        return role.getContainer().removeRole(role);
-    }
-
     @Override
     public PasswordPolicy getPasswordPolicy() {
         if (passwordPolicy == null) {
@@ -1932,12 +1925,6 @@ public class RealmAdapter implements RealmModel, JpaModel<RealmEntity> {
         return session.realms().createGroup(this, id, name);
     }
 
-    @Override
-    public void addTopLevelGroup(GroupModel subGroup) {
-        session.realms().addTopLevelGroup(this, subGroup);
-
-    }
-
     @Override
     public void moveGroup(GroupModel group, GroupModel toParent) {
         session.realms().moveGroup(this, group, toParent);

model/mongo/src/main/java/org/keycloak/models/mongo/keycloak/adapters/RealmAdapter.java

@@ -23,7 +23,6 @@ import org.keycloak.common.enums.SslRequired;
 import org.keycloak.common.util.MultivaluedHashMap;
 import org.keycloak.component.ComponentModel;
 import org.keycloak.connections.mongo.api.context.MongoStoreInvocationContext;
-import org.keycloak.jose.jwk.JWKBuilder;
 import org.keycloak.models.AuthenticationExecutionModel;
 import org.keycloak.models.AuthenticationFlowModel;
 import org.keycloak.models.AuthenticatorConfigModel;
@@ -62,10 +61,6 @@ import org.keycloak.models.mongo.keycloak.entities.UserFederationProviderEntity;
 import org.keycloak.models.utils.ComponentUtil;
 import org.keycloak.models.utils.KeycloakModelUtils;
 
-import java.security.Key;
-import java.security.PrivateKey;
-import java.security.PublicKey;
-import java.security.cert.X509Certificate;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -515,13 +510,6 @@ public class RealmAdapter extends AbstractMongoAdapter<MongoRealmEntity> impleme
         return session.realms().removeRole(this, role);
     }
 
-    @Override
-    public boolean removeRoleById(String id) {
-        RoleModel role = getRoleById(id);
-        if (role == null) return false;
-        return removeRole(role);
-    }
-
     @Override
     public Set<RoleModel> getRoles() {
         DBObject query = new QueryBuilder()
@@ -554,12 +542,6 @@ public class RealmAdapter extends AbstractMongoAdapter<MongoRealmEntity> impleme
         return session.realms().createGroup(this, id, name);
     }
 
-    @Override
-    public void addTopLevelGroup(GroupModel subGroup) {
-        session.realms().addTopLevelGroup(this, subGroup);
-
-    }
-
     @Override
     public void moveGroup(GroupModel group, GroupModel toParent) {
         session.realms().moveGroup(this, group, toParent);

pom.xml

@@ -633,6 +633,11 @@
                 <artifactId>infinispan-core</artifactId>
                 <version>${infinispan.version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.infinispan</groupId>
+                <artifactId>infinispan-cachestore-remote</artifactId>
+                <version>${infinispan.version}</version>
+            </dependency>
             <dependency>
                 <groupId>org.liquibase</groupId>
                 <artifactId>liquibase-core</artifactId>

server-spi-private/src/main/java/org/keycloak/cluster/ClusterListener.java

@@ -29,6 +29,6 @@ public interface ClusterListener {
      *
      * @param event value of notification (Object added into the cache)
      */
-    void run(ClusterEvent event);
+    void eventReceived(ClusterEvent event);
 
 }

server-spi-private/src/main/java/org/keycloak/cluster/ClusterProvider.java

@@ -48,7 +48,8 @@ public interface ClusterProvider extends Provider {
 
 
     /**
-     * Register task (listener) under given key. When this key will be put to the cache on any cluster node, the task will be executed
+     * Register task (listener) under given key. When this key will be put to the cache on any cluster node, the task will be executed.
+     * When using {@link #ALL} as the taskKey, then listener will be always triggered for any value put into the cache.
      *
      * @param taskKey
      * @param task
@@ -57,10 +58,18 @@ public interface ClusterProvider extends Provider {
 
 
     /**
-     * Notify registered listeners on all cluster nodes
+     * Notify registered listeners on all cluster nodes. It will notify listeners registered under given taskKey AND also listeners registered with {@link #ALL} key (those are always executed)
      *
      * @param taskKey
      * @param event
+     * @param ignoreSender if true, then sender node itself won't receive the notification
      */
-    void notify(String taskKey, ClusterEvent event);
+    void notify(String taskKey, ClusterEvent event, boolean ignoreSender);
+
+
+    /**
+     * Special value to be used with {@link #registerListener}  to specify that particular listener will be always triggered for all notifications
+     * with any key.
+     */
+    String ALL = "ALL";
 }

server-spi-private/src/main/java/org/keycloak/models/cache/CacheRealmProvider.java

@@ -27,12 +27,12 @@ public interface CacheRealmProvider extends RealmProvider {
     void clear();
     RealmProvider getDelegate();
 
-    void registerRealmInvalidation(String id);
+    void registerRealmInvalidation(String id, String name);
 
-    void registerClientInvalidation(String id);
+    void registerClientInvalidation(String id, String clientId, String realmId);
     void registerClientTemplateInvalidation(String id);
 
-    void registerRoleInvalidation(String id);
+    void registerRoleInvalidation(String id, String roleName, String roleContainerId);
 
     void registerGroupInvalidation(String id);
 }

server-spi/src/main/java/org/keycloak/models/RealmModel.java

@@ -351,8 +351,6 @@ public interface RealmModel extends RoleContainerModel {
 
     void setNotBefore(int notBefore);
 
-    boolean removeRoleById(String id);
-
     boolean isEventsEnabled();
 
     void setEventsEnabled(boolean enabled);
@@ -397,13 +395,6 @@ public interface RealmModel extends RoleContainerModel {
     GroupModel createGroup(String name);
     GroupModel createGroup(String id, String name);
 
-    /**
-     * Move Group to top realm level.  Basically just sets group parent to null.  You need to call this though
-     * to make sure caches are set properly
-     *
-     * @param subGroup
-     */
-    void addTopLevelGroup(GroupModel subGroup);
     GroupModel getGroupById(String id);
     List<GroupModel> getGroups();
     List<GroupModel> getTopLevelGroups();

services/src/main/java/org/keycloak/services/managers/UserStorageSyncManager.java

@@ -172,7 +172,7 @@ public class UserStorageSyncManager {
 
         }
         UserStorageProviderClusterEvent event = UserStorageProviderClusterEvent.createEvent(removed, realm.getId(), provider);
-        session.getProvider(ClusterProvider.class).notify(USER_STORAGE_TASK_KEY, event);
+        session.getProvider(ClusterProvider.class).notify(USER_STORAGE_TASK_KEY, event, false);
     }
 
 
@@ -282,7 +282,7 @@ public class UserStorageSyncManager {
         }
 
         @Override
-        public void run(ClusterEvent event) {
+        public void eventReceived(ClusterEvent event) {
             final UserStorageProviderClusterEvent fedEvent = (UserStorageProviderClusterEvent) event;
             KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
 

services/src/main/java/org/keycloak/services/managers/UsersSyncManager.java

@@ -155,7 +155,7 @@ public class UsersSyncManager {
     // Ensure all cluster nodes are notified
     public void notifyToRefreshPeriodicSync(KeycloakSession session, RealmModel realm, UserFederationProviderModel federationProvider, boolean removed) {
         FederationProviderClusterEvent event = FederationProviderClusterEvent.createEvent(removed, realm.getId(), federationProvider);
-        session.getProvider(ClusterProvider.class).notify(FEDERATION_TASK_KEY, event);
+        session.getProvider(ClusterProvider.class).notify(FEDERATION_TASK_KEY, event, false);
     }
 
 
@@ -265,7 +265,7 @@ public class UsersSyncManager {
         }
 
         @Override
-        public void run(ClusterEvent event) {
+        public void eventReceived(ClusterEvent event) {
             final FederationProviderClusterEvent fedEvent = (FederationProviderClusterEvent) event;
             KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
 

services/src/main/java/org/keycloak/services/resources/KeycloakApplication.java

@@ -51,6 +51,7 @@ import org.keycloak.services.resources.admin.AdminRoot;
 import org.keycloak.services.scheduled.ClearExpiredEvents;
 import org.keycloak.services.scheduled.ClearExpiredUserSessions;
 import org.keycloak.services.scheduled.ClusterAwareScheduledTaskRunner;
+import org.keycloak.services.scheduled.ScheduledTaskRunner;
 import org.keycloak.services.util.JsonConfigProvider;
 import org.keycloak.services.util.ObjectMapperResolver;
 import org.keycloak.timer.TimerProvider;
@@ -321,7 +322,7 @@ public class KeycloakApplication extends Application {
         try {
             TimerProvider timer = session.getProvider(TimerProvider.class);
             timer.schedule(new ClusterAwareScheduledTaskRunner(sessionFactory, new ClearExpiredEvents(), interval), interval, "ClearExpiredEvents");
-            timer.schedule(new ClusterAwareScheduledTaskRunner(sessionFactory, new ClearExpiredUserSessions(), interval), interval, "ClearExpiredUserSessions");
+            timer.schedule(new ScheduledTaskRunner(sessionFactory, new ClearExpiredUserSessions()), interval, "ClearExpiredUserSessions");
             new UsersSyncManager().bootstrapPeriodic(sessionFactory, timer);
             new UserStorageSyncManager().bootstrapPeriodic(sessionFactory, timer);
         } finally {

testsuite/integration/pom.xml

@@ -232,6 +232,10 @@
             <groupId>org.infinispan</groupId>
             <artifactId>infinispan-core</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.infinispan</groupId>
+            <artifactId>infinispan-cachestore-remote</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.seleniumhq.selenium</groupId>
             <artifactId>selenium-java</artifactId>

testsuite/integration/src/test/java/org/keycloak/testsuite/federation/storage/ldap/LDAPGroupMapperSyncTest.java

@@ -238,7 +238,7 @@ public class LDAPGroupMapperSyncTest {
             GroupModel model1 = realm.createGroup("model1");
             realm.moveGroup(model1, null);
             GroupModel model2 = realm.createGroup("model2");
-            kcGroup1.addChild(model2);
+            realm.moveGroup(model2, kcGroup1);
 
             // Sync groups again from LDAP. Nothing deleted
             syncResult = new GroupLDAPStorageMapperFactory().create(session, mapperModel).syncDataFromFederationProviderToKeycloak(mapperModel, ldapProvider, session, realm);

testsuite/integration/src/test/java/org/keycloak/testsuite/model/ClusterInvalidationTest.java

@@ -0,0 +1,420 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.testsuite.model;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.infinispan.Cache;
+import org.infinispan.notifications.Listener;
+import org.infinispan.notifications.cachelistener.annotation.CacheEntryRemoved;
+import org.infinispan.notifications.cachelistener.event.CacheEntryRemovedEvent;
+import org.jboss.logging.Logger;
+import org.junit.Assert;
+import org.junit.ClassRule;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
+import org.keycloak.models.ClientModel;
+import org.keycloak.models.ClientTemplateModel;
+import org.keycloak.models.GroupModel;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.RealmModel;
+import org.keycloak.models.RoleModel;
+import org.keycloak.models.UserConsentModel;
+import org.keycloak.models.UserModel;
+import org.keycloak.models.utils.KeycloakModelUtils;
+import org.keycloak.services.managers.RealmManager;
+import org.keycloak.testsuite.KeycloakServer;
+import org.keycloak.testsuite.rule.KeycloakRule;
+import org.keycloak.testsuite.util.cli.TestCacheUtils;
+
+/**
+ * Requires execution with cluster (or external JDG) enabled and real database, which will be shared for both cluster nodes. Everything set by system properties:
+ *
+ * 1) Use those system properties to run against shared MySQL:
+ *
+ *  -Dkeycloak.connectionsJpa.url=jdbc:mysql://localhost/keycloak -Dkeycloak.connectionsJpa.driver=com.mysql.jdbc.Driver -Dkeycloak.connectionsJpa.user=keycloak
+ *  -Dkeycloak.connectionsJpa.password=keycloak
+ *
+ *
+ * 2) Then either choose from:
+ *
+ * 2.a) Run test with 2 keycloak nodes in cluster. Add this system property for that: -Dkeycloak.connectionsInfinispan.clustered=true
+ *
+ * 2.b) Run test with 2 keycloak nodes without cluster, but instead with external JDG. Both keycloak servers will send invalidation events to the JDG server and receive the events from this JDG server.
+ * They don't communicate with each other. So JDG is man-in-the-middle.
+ *
+ * This assumes that you have JDG 7.0 server running on localhost with HotRod endpoint on port 11222 (which is default port anyway).
+ *
+ * You also need to have this cache configured in JDG_HOME/standalone/configuration/standalone.xml to infinispan subsystem :
+ *
+ *  <local-cache name="work" start="EAGER" batching="false" />
+ *
+ * Finally, add this system property when running the test: -Dkeycloak.connectionsInfinispan.remoteStoreEnabled=true
+ *
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+@Ignore
+public class ClusterInvalidationTest {
+
+    protected static final Logger logger = Logger.getLogger(ClusterInvalidationTest.class);
+
+    private static final String REALM_NAME = "test";
+
+    private static final int SLEEP_TIME_MS = Integer.parseInt(System.getProperty("sleep.time", "500"));
+
+    private static TestListener listener1realms;
+    private static TestListener listener1users;
+    private static TestListener listener2realms;
+    private static TestListener listener2users;
+
+    @ClassRule
+    public static KeycloakRule server1 = new KeycloakRule(new KeycloakRule.KeycloakSetup() {
+
+        @Override
+        public void config(RealmManager manager, RealmModel adminstrationRealm, RealmModel appRealm) {
+            InfinispanConnectionProvider infinispan = manager.getSession().getProvider(InfinispanConnectionProvider.class);
+
+            Cache cache = infinispan.getCache(InfinispanConnectionProvider.REALM_CACHE_NAME);
+            listener1realms = new TestListener("server1 - realms", cache);
+            cache.addListener(listener1realms);
+
+            cache = infinispan.getCache(InfinispanConnectionProvider.USER_CACHE_NAME);
+            listener1users = new TestListener("server1 - users", cache);
+            cache.addListener(listener1users);
+        }
+
+    });
+
+    @ClassRule
+    public static KeycloakRule server2 = new KeycloakRule(new KeycloakRule.KeycloakSetup() {
+
+        @Override
+        public void config(RealmManager manager, RealmModel adminstrationRealm, RealmModel appRealm) {
+            InfinispanConnectionProvider infinispan = manager.getSession().getProvider(InfinispanConnectionProvider.class);
+
+            Cache cache = infinispan.getCache(InfinispanConnectionProvider.REALM_CACHE_NAME);
+            listener2realms = new TestListener("server2 - realms", cache);
+            cache.addListener(listener2realms);
+
+            cache = infinispan.getCache(InfinispanConnectionProvider.USER_CACHE_NAME);
+            listener2users = new TestListener("server2 - users", cache);
+            cache.addListener(listener2users);
+        }
+
+    }) {
+
+        @Override
+        protected void configureServer(KeycloakServer server) {
+            server.getConfig().setPort(8082);
+        }
+
+        @Override
+        protected void importRealm() {
+        }
+
+        @Override
+        protected void removeTestRealms() {
+        }
+
+    };
+
+    private static void clearListeners() {
+        listener1realms.getInvalidationsAndClear();
+        listener1users.getInvalidationsAndClear();
+        listener2realms.getInvalidationsAndClear();
+        listener2users.getInvalidationsAndClear();
+    }
+
+
+    @Test
+    public void testClusterInvalidation() throws Exception {
+        cacheEverything();
+
+        clearListeners();
+
+        KeycloakSession session1 = server1.startSession();
+
+
+        logger.info("UPDATE REALM");
+
+        RealmModel realm = session1.realms().getRealmByName(REALM_NAME);
+        realm.setDisplayName("foo");
+        session1 = commit(server1, session1, true);
+
+        assertInvalidations(listener1realms.getInvalidationsAndClear(), 1, 3, realm.getId());
+        assertInvalidations(listener2realms.getInvalidationsAndClear(), 1, 3, realm.getId());
+
+
+        // CREATES
+
+        logger.info("CREATE ROLE");
+        realm = session1.realms().getRealmByName(REALM_NAME);
+        realm.addRole("foo-role");
+        session1 = commit(server1, session1, true);
+
+        assertInvalidations(listener1realms.getInvalidationsAndClear(), 1, 1, "test.roles");
+        assertInvalidations(listener2realms.getInvalidationsAndClear(), 1, 1, "test.roles");
+
+
+        logger.info("CREATE CLIENT");
+        realm = session1.realms().getRealmByName(REALM_NAME);
+        realm.addClient("foo-client");
+        session1 = commit(server1, session1, true);
+
+        assertInvalidations(listener1realms.getInvalidationsAndClear(), 1, 1, "test.realm.clients");
+        assertInvalidations(listener2realms.getInvalidationsAndClear(), 1, 1, "test.realm.clients");
+
+        logger.info("CREATE GROUP");
+        realm = session1.realms().getRealmByName(REALM_NAME);
+        GroupModel group = realm.createGroup("foo-group");
+        session1 = commit(server1, session1, true);
+
+        assertInvalidations(listener1realms.getInvalidationsAndClear(), 1, 1, "test.top.groups");
+        assertInvalidations(listener2realms.getInvalidationsAndClear(), 1, 1, "test.top.groups");
+
+        logger.info("CREATE CLIENT TEMPLATE");
+        realm = session1.realms().getRealmByName(REALM_NAME);
+        realm.addClientTemplate("foo-template");
+        session1 = commit(server1, session1, true);
+
+        assertInvalidations(listener1realms.getInvalidationsAndClear(), 2, 3, realm.getId());
+        assertInvalidations(listener2realms.getInvalidationsAndClear(), 0, 2); // realm not cached on server2 due to previous invalidation
+
+
+        // UPDATES
+
+        logger.info("UPDATE ROLE");
+        realm = session1.realms().getRealmByName(REALM_NAME);
+        ClientModel testApp = realm.getClientByClientId("test-app");
+        RoleModel role = session1.realms().getClientRole(realm, testApp, "customer-user");
+        role.setDescription("Foo");
+        session1 = commit(server1, session1, true);
+
+        assertInvalidations(listener1realms.getInvalidationsAndClear(), 2, 3, role.getId());
+        assertInvalidations(listener2realms.getInvalidationsAndClear(), 2, 3, role.getId());
+
+        logger.info("UPDATE GROUP");
+        realm = session1.realms().getRealmByName(REALM_NAME);
+        group = KeycloakModelUtils.findGroupByPath(realm, "/topGroup");
+        group.grantRole(role);
+        session1 = commit(server1, session1, true);
+
+        assertInvalidations(listener1realms.getInvalidationsAndClear(), 1, 1, group.getId());
+        assertInvalidations(listener2realms.getInvalidationsAndClear(), 1, 1, group.getId());
+
+        logger.info("UPDATE CLIENT");
+        realm = session1.realms().getRealmByName(REALM_NAME);
+        testApp = realm.getClientByClientId("test-app");
+        testApp.setDescription("foo");;
+        session1 = commit(server1, session1, true);
+
+        assertInvalidations(listener1realms.getInvalidationsAndClear(), 2, 3, testApp.getId());
+        assertInvalidations(listener2realms.getInvalidationsAndClear(), 2, 3, testApp.getId());
+
+        // Cache client template on server2
+        KeycloakSession session2 = server2.startSession();
+        realm = session2.realms().getRealmByName(REALM_NAME);
+        realm.getClientTemplates().get(0);
+
+
+        logger.info("UPDATE CLIENT TEMPLATE");
+        realm = session1.realms().getRealmByName(REALM_NAME);
+        ClientTemplateModel clientTemplate = realm.getClientTemplates().get(0);
+        clientTemplate.setDescription("bar");
+
+        session1 = commit(server1, session1, true);
+
+        assertInvalidations(listener1realms.getInvalidationsAndClear(), 1, 1, clientTemplate.getId());
+        assertInvalidations(listener2realms.getInvalidationsAndClear(), 1, 1, clientTemplate.getId());
+
+        // Nothing yet invalidated in user cache
+        assertInvalidations(listener1users.getInvalidationsAndClear(), 0, 0);
+        assertInvalidations(listener2users.getInvalidationsAndClear(), 0, 0);
+
+        logger.info("UPDATE USER");
+        realm = session1.realms().getRealmByName(REALM_NAME);
+        UserModel user = session1.users().getUserByEmail("keycloak-user@localhost", realm);
+        user.setSingleAttribute("foo", "Bar");
+        session1 = commit(server1, session1, true);
+
+        assertInvalidations(listener1users.getInvalidationsAndClear(), 1, 5, user.getId(), "test.email.keycloak-user@localhost");
+        assertInvalidations(listener2users.getInvalidationsAndClear(), 1, 5, user.getId());
+
+        logger.info("UPDATE USER CONSENTS");
+        realm = session1.realms().getRealmByName(REALM_NAME);
+        testApp = realm.getClientByClientId("test-app");
+        user = session1.users().getUserByEmail("keycloak-user@localhost", realm);
+        session1.users().addConsent(realm, user.getId(), new UserConsentModel(testApp));
+        session1 = commit(server1, session1, true);
+
+        assertInvalidations(listener1users.getInvalidationsAndClear(), 1, 1, user.getId() + ".consents");
+        assertInvalidations(listener2users.getInvalidationsAndClear(), 1, 1, user.getId() + ".consents");
+
+
+        // REMOVALS
+
+        logger.info("REMOVE USER");
+        realm = session1.realms().getRealmByName(REALM_NAME);
+        user = session1.users().getUserByUsername("john-doh@localhost", realm);
+        session1.users().removeUser(realm, user);
+
+        session1 = commit(server1, session1, true);
+
+        assertInvalidations(listener1users.getInvalidationsAndClear(), 3, 5, user.getId(), user.getId() + ".consents", "test.username.john-doh@localhost");
+        assertInvalidations(listener2users.getInvalidationsAndClear(), 2, 5, user.getId(), user.getId() + ".consents");
+
+        cacheEverything();
+
+        logger.info("REMOVE CLIENT TEMPLATE");
+        realm = session1.realms().getRealmByName(REALM_NAME);
+        realm.removeClientTemplate(clientTemplate.getId());
+        session1 = commit(server1, session1, true);
+
+        assertInvalidations(listener1realms.getInvalidationsAndClear(), 2, 5, realm.getId(), clientTemplate.getId());
+        assertInvalidations(listener2realms.getInvalidationsAndClear(), 2, 5, realm.getId(), clientTemplate.getId());
+
+        cacheEverything();
+
+        logger.info("REMOVE ROLE");
+        realm = session1.realms().getRealmByName(REALM_NAME);
+        role = realm.getRole("user");
+        realm.removeRole(role);
+        ClientModel thirdparty = session1.realms().getClientByClientId("third-party", realm);
+        session1 = commit(server1, session1, true);
+
+        assertInvalidations(listener1realms.getInvalidationsAndClear(), 7, 10, role.getId(), realm.getId(), "test.roles", "test.user.roles", testApp.getId(), thirdparty.getId(), group.getId());
+        assertInvalidations(listener2realms.getInvalidationsAndClear(), 7, 10, role.getId(), realm.getId(), "test.roles", "test.user.roles", testApp.getId(), thirdparty.getId(), group.getId());
+
+        // all users invalidated
+        assertInvalidations(listener1users.getInvalidationsAndClear(), 10, 100);
+        assertInvalidations(listener2users.getInvalidationsAndClear(), 10, 100);
+
+        cacheEverything();
+
+        logger.info("REMOVE GROUP");
+        realm = session1.realms().getRealmByName(REALM_NAME);
+        group = realm.getGroupById(group.getId());
+        String subgroupId = group.getSubGroups().iterator().next().getId();
+        realm.removeGroup(group);
+        session1 = commit(server1, session1, true);
+
+        assertInvalidations(listener1realms.getInvalidationsAndClear(), 3, 5, group.getId(), subgroupId, "test.top.groups");
+        assertInvalidations(listener2realms.getInvalidationsAndClear(), 3, 5, group.getId(), subgroupId, "test.top.groups");
+
+        // all users invalidated
+        assertInvalidations(listener1users.getInvalidationsAndClear(), 10, 100);
+        assertInvalidations(listener2users.getInvalidationsAndClear(), 10, 100);
+
+        cacheEverything();
+
+        logger.info("REMOVE CLIENT");
+        realm = session1.realms().getRealmByName(REALM_NAME);
+        testApp = realm.getClientByClientId("test-app");
+        role = testApp.getRole("customer-user");
+        realm.removeClient(testApp.getId());
+        session1 = commit(server1, session1, true);
+
+        assertInvalidations(listener1realms.getInvalidationsAndClear(), 8, 12, testApp.getId(), testApp.getId() + ".roles", role.getId(), testApp.getId() + ".customer-user.roles", "test.realm.clients", thirdparty.getId());
+        assertInvalidations(listener2realms.getInvalidationsAndClear(), 8, 12, testApp.getId(), testApp.getId() + ".roles", role.getId(), testApp.getId() + ".customer-user.roles", "test.realm.clients", thirdparty.getId());
+
+        // all users invalidated
+        assertInvalidations(listener1users.getInvalidationsAndClear(), 10, 100);
+        assertInvalidations(listener2users.getInvalidationsAndClear(), 10, 100);
+
+        cacheEverything();
+
+        logger.info("REMOVE REALM");
+        realm = session1.realms().getRealmByName(REALM_NAME);
+        session1.realms().removeRealm(realm.getId());
+        session1 = commit(server1, session1, true);
+
+        assertInvalidations(listener1realms.getInvalidationsAndClear(), 50, 200, realm.getId(), thirdparty.getId());
+        assertInvalidations(listener2realms.getInvalidationsAndClear(), 50, 200, realm.getId(), thirdparty.getId());
+
+        // all users invalidated
+        assertInvalidations(listener1users.getInvalidationsAndClear(), 10, 100);
+        assertInvalidations(listener2users.getInvalidationsAndClear(), 10, 100);
+
+
+        //Thread.sleep(10000000);
+    }
+
+    private void assertInvalidations(Map<String, Object> invalidations, int low, int high, String... expectedNames) {
+        int size = invalidations.size();
+        Assert.assertTrue("Size was " + size + ". Entries were: " + invalidations.keySet(), size >= low);
+        Assert.assertTrue("Size was " + size + ". Entries were: " + invalidations.keySet(), size <= high);
+
+        for (String expected : expectedNames) {
+            Assert.assertTrue("Can't find " + expected + ". Entries were: " + invalidations.keySet(), invalidations.keySet().contains(expected));
+        }
+    }
+
+    private KeycloakSession commit(KeycloakRule rule, KeycloakSession session, boolean sleepAfterCommit) throws Exception {
+        session.getTransactionManager().commit();
+        session.close();
+
+        if (sleepAfterCommit) {
+            Thread.sleep(SLEEP_TIME_MS);
+        }
+
+        return rule.startSession();
+    }
+
+    private void cacheEverything() throws Exception {
+        KeycloakSession session1 = server1.startSession();
+        TestCacheUtils.cacheRealmWithEverything(session1, REALM_NAME);
+        session1 = commit(server1, session1, false);
+
+        KeycloakSession session2 = server2.startSession();
+        TestCacheUtils.cacheRealmWithEverything(session2, REALM_NAME);
+        session2 = commit(server1, session2, false);
+    }
+
+
+    @Listener(observation = Listener.Observation.PRE)
+    public static class TestListener {
+
+        private final String name;
+        private final Cache cache; // Just for debugging
+
+        private Map<String, Object> invalidations = new ConcurrentHashMap<>();
+
+        public TestListener(String name, Cache cache) {
+            this.name = name;
+            this.cache = cache;
+        }
+
+        @CacheEntryRemoved
+        public void cacheEntryRemoved(CacheEntryRemovedEvent event) {
+            logger.infof("%s: Invalidated %s: %s", name, event.getKey(), event.getValue());
+            invalidations.put(event.getKey().toString(), event.getValue());
+        }
+
+        Map<String, Object> getInvalidationsAndClear() {
+            Map<String, Object> newMap = new HashMap<>(invalidations);
+            invalidations.clear();
+            return newMap;
+        }
+
+    }
+
+
+}

testsuite/integration/src/test/java/org/keycloak/testsuite/util/cli/CacheCommands.java

@@ -0,0 +1,115 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.testsuite.util.cli;
+
+import java.util.Map;
+import java.util.Set;
+
+import org.infinispan.Cache;
+import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.RealmModel;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class CacheCommands {
+
+    public static class ListCachesCommand extends AbstractCommand {
+
+        @Override
+        public String getName() {
+            return "listCaches";
+        }
+
+        @Override
+        protected void doRunCommand(KeycloakSession session) {
+            InfinispanConnectionProvider ispnProvider = session.getProvider(InfinispanConnectionProvider.class);
+            Set<String> cacheNames = ispnProvider.getCache("realms").getCacheManager().getCacheNames();
+            log.infof("Available caches: %s", cacheNames);
+        }
+
+    }
+
+
+    public static class GetCacheCommand extends AbstractCommand {
+
+        @Override
+        public String getName() {
+            return "getCache";
+        }
+
+        @Override
+        protected void doRunCommand(KeycloakSession session) {
+            String cacheName = getArg(0);
+            InfinispanConnectionProvider ispnProvider = session.getProvider(InfinispanConnectionProvider.class);
+            Cache<Object, Object> cache = ispnProvider.getCache(cacheName);
+            if (cache == null) {
+                log.errorf("Cache '%s' doesn't exist", cacheName);
+                throw new HandledException();
+            }
+
+            printCache(cache);
+        }
+
+        private void printCache(Cache<Object, Object> cache) {
+            int size = cache.size();
+            log.infof("Cache %s, size: %d", cache.getName(), size);
+
+            if (size > 50) {
+                log.info("Skip printing cache recors due to big size");
+            } else {
+                for (Map.Entry<Object, Object> entry : cache.entrySet()) {
+                    log.infof("%s=%s", entry.getKey(), entry.getValue());
+                }
+            }
+        }
+
+        @Override
+        public String printUsage() {
+            return super.printUsage() + " <cache-name> . cache-name is name of the infinispan cache provided by InfinispanConnectionProvider";
+        }
+
+    }
+
+
+    public static class CacheRealmObjectsCommand extends AbstractCommand {
+
+        @Override
+        public String getName() {
+            return "cacheRealmObjects";
+        }
+
+        @Override
+        protected void doRunCommand(KeycloakSession session) {
+            String realmName = getArg(0);
+            RealmModel realm = session.realms().getRealmByName(realmName);
+            if (realm == null) {
+                log.errorf("Realm not found: %s", realmName);
+                throw new HandledException();
+            }
+
+            TestCacheUtils.cacheRealmWithEverything(session, realmName);
+        }
+
+        @Override
+        public String printUsage() {
+            return super.printUsage() + " <realm-name>";
+        }
+    }
+}

testsuite/integration/src/test/java/org/keycloak/testsuite/util/cli/RoleCommands.java

@@ -0,0 +1,127 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.testsuite.util.cli;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.keycloak.models.ClientModel;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.KeycloakSessionTask;
+import org.keycloak.models.RealmModel;
+import org.keycloak.models.RoleContainerModel;
+import org.keycloak.models.RoleModel;
+import org.keycloak.models.utils.KeycloakModelUtils;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class RoleCommands {
+
+    public static class CreateRoles extends AbstractCommand {
+
+        private String rolePrefix;
+        private String roleContainer;
+
+        @Override
+        public String getName() {
+            return "createRoles";
+        }
+
+        private class StateHolder {
+            int firstInThisBatch;
+            int countInThisBatch;
+            int remaining;
+        };
+
+        @Override
+        protected void doRunCommand(KeycloakSession session) {
+            rolePrefix = getArg(0);
+            roleContainer = getArg(1);
+            int first = getIntArg(2);
+            int count = getIntArg(3);
+            int batchCount = getIntArg(4);
+
+            final StateHolder state = new StateHolder();
+            state.firstInThisBatch = first;
+            state.remaining = count;
+            state.countInThisBatch = Math.min(batchCount, state.remaining);
+            while (state.remaining > 0) {
+                KeycloakModelUtils.runJobInTransaction(session.getKeycloakSessionFactory(), new KeycloakSessionTask() {
+
+                    @Override
+                    public void run(KeycloakSession session) {
+                        createRolesInBatch(session, roleContainer, rolePrefix, state.firstInThisBatch, state.countInThisBatch);
+                    }
+                });
+
+                // update state
+                state.firstInThisBatch = state.firstInThisBatch + state.countInThisBatch;
+                state.remaining = state.remaining - state.countInThisBatch;
+                state.countInThisBatch = Math.min(batchCount, state.remaining);
+            }
+
+            log.infof("Command finished. All roles from %s to %s created", rolePrefix + first, rolePrefix + (first + count - 1));
+        }
+
+        private void createRolesInBatch(KeycloakSession session, String roleContainer, String rolePrefix, int first, int count) {
+            RoleContainerModel container = getRoleContainer(session, roleContainer);
+
+            int last = first + count;
+            for (int counter = first; counter < last; counter++) {
+                String roleName = rolePrefix + counter;
+                RoleModel role = container.addRole(roleName);
+            }
+            log.infof("Roles from %s to %s created", rolePrefix + first, rolePrefix + (last - 1));
+        }
+
+        private RoleContainerModel getRoleContainer(KeycloakSession session, String roleContainer) {
+            String[] parts = roleContainer.split("/");
+            String realmName = parts[0];
+
+            RealmModel realm = session.realms().getRealmByName(realmName);
+            if (realm == null) {
+                log.errorf("Unknown realm: %s", realmName);
+                throw new HandledException();
+            }
+
+            if (parts.length == 1) {
+                return realm;
+            } else {
+                String clientId = parts[1];
+                ClientModel client = session.realms().getClientByClientId(clientId, realm);
+                if (client == null) {
+                    log.errorf("Unknown client: %s", clientId);
+                    throw new HandledException();
+                }
+
+                return client;
+            }
+        }
+
+        @Override
+        public String printUsage() {
+            return super.printUsage() + " <role-prefix> <role-container> <starting-role-offset> <total-count> <batch-size> . " +
+                    "\n'total-count' refers to total count of newly created roles. 'batch-size' refers to number of created roles in each transaction. 'starting-role-offset' refers to starting role offset." +
+                    "\nFor example if 'starting-role-offset' is 15 and total-count is 10 and role-prefix is 'test', it will create roles test15, test16, test17, ... , test24" +
+                    "\n'role-container' is either realm (then use just realmName like 'demo' or client (then use realm/clientId like 'demo/my-client' .\n" +
+                    "Example usage: " + super.printUsage() + " test demo 0 500 100";
+        }
+
+    }
+}

testsuite/integration/src/test/java/org/keycloak/testsuite/util/cli/TestCacheUtils.java

@@ -0,0 +1,88 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.testsuite.util.cli;
+
+import org.keycloak.models.ClientModel;
+import org.keycloak.models.ClientTemplateModel;
+import org.keycloak.models.FederatedIdentityModel;
+import org.keycloak.models.GroupModel;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.RealmModel;
+import org.keycloak.models.RoleContainerModel;
+import org.keycloak.models.RoleModel;
+import org.keycloak.models.UserModel;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class TestCacheUtils {
+
+    public static void cacheRealmWithEverything(KeycloakSession session, String realmName) {
+        RealmModel realm  = session.realms().getRealmByName(realmName);
+
+        for (ClientModel client : realm.getClients()) {
+            realm.getClientById(client.getId());
+            realm.getClientByClientId(client.getClientId());
+
+            cacheRoles(session, realm, client);
+        }
+
+        cacheRoles(session, realm, realm);
+
+        for (GroupModel group : realm.getTopLevelGroups()) {
+            cacheGroupRecursive(realm, group);
+        }
+
+        for (ClientTemplateModel clientTemplate : realm.getClientTemplates()) {
+            realm.getClientTemplateById(clientTemplate.getId());
+        }
+
+        for (UserModel user : session.users().getUsers(realm)) {
+            session.users().getUserById(user.getId(), realm);
+            if (user.getEmail() != null) {
+                session.users().getUserByEmail(user.getEmail(), realm);
+            }
+            session.users().getUserByUsername(user.getUsername(), realm);
+
+            session.users().getConsents(realm, user.getId());
+
+            for (FederatedIdentityModel fedIdentity : session.users().getFederatedIdentities(user, realm)) {
+                session.users().getUserByFederatedIdentity(fedIdentity, realm);
+            }
+        }
+    }
+
+    private static void cacheRoles(KeycloakSession session, RealmModel realm, RoleContainerModel roleContainer) {
+        for (RoleModel role : roleContainer.getRoles()) {
+            realm.getRoleById(role.getId());
+            roleContainer.getRole(role.getName());
+            if (roleContainer instanceof RealmModel) {
+                session.realms().getRealmRole(realm, role.getName());
+            } else {
+                session.realms().getClientRole(realm, (ClientModel) roleContainer, role.getName());
+            }
+        }
+    }
+
+    private static void cacheGroupRecursive(RealmModel realm, GroupModel group) {
+        realm.getGroupById(group.getId());
+        for (GroupModel sub : group.getSubGroups()) {
+            cacheGroupRecursive(realm, sub);
+        }
+    }
+}

testsuite/integration/src/test/java/org/keycloak/testsuite/util/cli/TestsuiteCLI.java

@@ -57,7 +57,11 @@ public class TestsuiteCLI {
             UserCommands.Remove.class,
             UserCommands.Count.class,
             UserCommands.GetUser.class,
-            SyncDummyFederationProviderCommand.class
+            SyncDummyFederationProviderCommand.class,
+            RoleCommands.CreateRoles.class,
+            CacheCommands.ListCachesCommand.class,
+            CacheCommands.GetCacheCommand.class,
+            CacheCommands.CacheRealmObjectsCommand.class
     };
 
     private final KeycloakSessionFactory sessionFactory;

testsuite/integration/src/test/resources/META-INF/keycloak-server.json

@@ -97,7 +97,10 @@
         "default": {
             "clustered": "${keycloak.connectionsInfinispan.clustered:false}",
             "async": "${keycloak.connectionsInfinispan.async:false}",
-            "sessionsOwners": "${keycloak.connectionsInfinispan.sessionsOwners:2}"
+            "sessionsOwners": "${keycloak.connectionsInfinispan.sessionsOwners:2}",
+            "remoteStoreEnabled": "${keycloak.connectionsInfinispan.remoteStoreEnabled:false}",
+            "remoteStoreHost": "${keycloak.connectionsInfinispan.remoteStoreHost:localhost}",
+            "remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort:11222}"
         }
     },
 

testsuite/integration/src/test/resources/log4j.properties

@@ -46,7 +46,8 @@ log4j.logger.org.keycloak.connections.jpa.updater.liquibase=${keycloak.liquibase
 # log4j.logger.org.keycloak.models.sessions.infinispan.initializer=trace
 
 # Enable to view cache activity
-# log4j.logger.org.keycloak.models.cache=trace
+#log4j.logger.org.keycloak.cluster.infinispan=trace
+#log4j.logger.org.keycloak.models.cache.infinispan=debug
 
 # Enable to view database updates
 # log4j.logger.org.keycloak.connections.mongo.updater.DefaultMongoUpdaterProvider=debug

wildfly/server-subsystem/src/main/resources/subsystem-templates/keycloak-infinispan.xml

@@ -92,10 +92,10 @@
         <replacement placeholder="CACHE-CONTAINERS">
             <cache-container name="keycloak" jndi-name="infinispan/Keycloak">
                 <transport lock-timeout="60000"/>
-                <invalidation-cache name="realms" mode="SYNC"/>
+                <local-cache name="realms"/>
-                <invalidation-cache name="users" mode="SYNC">
+                <local-cache name="users">
                     <eviction max-entries="10000" strategy="LRU"/>
-                </invalidation-cache>
+                </local-cache>
                 <distributed-cache name="sessions" mode="SYNC" owners="1"/>
                 <distributed-cache name="offlineSessions" mode="SYNC" owners="1"/>
                 <distributed-cache name="loginFailures" mode="SYNC" owners="1"/>