KEYCLOAK-7594 Upgrade to Wildfly 13. Cross-DC: Upgrade to infinispan server 9.2.4 and JDG 7.2

Co-authored-by: Douglas Palmer <dpalmer@redhat.com>
Co-authored-by: stianst <stianst@gmail.com>
Co-authored-by: Hynek Mlnarik <hmlnarik@redhat.com>

adapters/oidc/installed/pom.xml

@@ -69,7 +69,7 @@
         </dependency>
         <dependency>
             <groupId>org.jboss.spec.javax.ws.rs</groupId>
-            <artifactId>jboss-jaxrs-api_2.0_spec</artifactId>
+            <artifactId>jboss-jaxrs-api_2.1_spec</artifactId>
         </dependency>
 
     </dependencies>

adapters/oidc/jaxrs-oauth-client/pom.xml

@@ -33,7 +33,7 @@
     <dependencies>
         <dependency>
             <groupId>org.jboss.spec.javax.ws.rs</groupId>
-            <artifactId>jboss-jaxrs-api_2.0_spec</artifactId>
+            <artifactId>jboss-jaxrs-api_2.1_spec</artifactId>
             <scope>provided</scope>
         </dependency>
         <dependency>

authz/policy/drools/pom.xml

@@ -55,7 +55,7 @@
 
         <dependency>
             <groupId>org.jboss.spec.javax.ws.rs</groupId>
-            <artifactId>jboss-jaxrs-api_2.0_spec</artifactId>
+            <artifactId>jboss-jaxrs-api_2.1_spec</artifactId>
             <scope>provided</scope>
         </dependency>
     </dependencies>

distribution/demo-dist/src/main/xslt/standalone.xsl

@@ -17,7 +17,7 @@
 
 <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
                 xmlns:xalan="http://xml.apache.org/xalan"
-                xmlns:j="urn:jboss:domain:5.0"
+                xmlns:j="urn:jboss:domain:7.0"
                 xmlns:ds="urn:jboss:domain:datasources:5.0"
                 xmlns:k="urn:jboss:domain:keycloak:1.1"
                 xmlns:sec="urn:jboss:domain:security:2.0"
@@ -81,12 +81,12 @@
 
     <xsl:template match="//*[local-name()='subsystem' and starts-with(namespace-uri(), $inf)]">
         <xsl:copy>
-            <cache-container name="keycloak" jndi-name="infinispan/Keycloak">
+            <cache-container name="keycloak">
                 <local-cache name="realms">
-                    <eviction max-entries="10000" strategy="LRU"/>
+                    <object-memory size="10000"/>
                 </local-cache>
                 <local-cache name="users">
-                    <eviction max-entries="10000" strategy="LRU"/>
+                    <object-memory size="10000"/>
                 </local-cache>
                 <local-cache name="sessions"/>
                 <local-cache name="authenticationSessions"/>
@@ -95,12 +95,12 @@
                 <local-cache name="offlineClientSessions"/>
                 <local-cache name="loginFailures"/>
                 <local-cache name="authorization">
-                    <eviction max-entries="10000" strategy="LRU"/>
+                    <object-memory size="10000"/>
                 </local-cache>
                 <local-cache name="actionTokens"/>
                 <local-cache name="work"/>
                 <local-cache name="keys">
-                    <eviction max-entries="1000" strategy="LRU"/>
+                    <object-memory size="1000"/>
                     <expiration max-idle="3600000" />
                 </local-cache>
             </cache-container>

distribution/feature-packs/server-feature-pack/assembly.xml

@@ -33,6 +33,10 @@
             <directory>target/unpacked-themes/theme</directory>
             <outputDirectory>content/themes</outputDirectory>
         </fileSet>
+        <fileSet>
+            <directory>target/keycloak-client-tools/bin</directory>
+            <outputDirectory>content/bin</outputDirectory>
+        </fileSet>
         <fileSet>
             <directory>src/main/resources/identity/module</directory>
             <includes>

distribution/feature-packs/server-feature-pack/pom.xml

@@ -612,6 +612,17 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>org.keycloak</groupId>
+            <artifactId>keycloak-client-cli-dist</artifactId>
+            <type>zip</type>
+            <exclusions>
+                <exclusion>
+                    <groupId>*</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
         <dependency>
             <groupId>org.kie</groupId>
             <artifactId>kie-api</artifactId>
@@ -713,6 +724,16 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>org.aesh</groupId>
+            <artifactId>aesh</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>*</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
     </dependencies>
 
     <build>
@@ -740,9 +761,34 @@
                 </executions>
             </plugin>
 
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-dependency-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>unpack-cli</id>
+                        <phase>validate</phase>
+                        <goals>
+                            <goal>unpack</goal>
+                        </goals>
+                        <configuration>
+                            <artifactItems>
+                                <artifactItem>
+                                    <groupId>org.keycloak</groupId>
+                                    <artifactId>keycloak-client-cli-dist</artifactId>
+                                    <type>zip</type>
+                                    <outputDirectory>target/</outputDirectory>
+                                </artifactItem>
+                            </artifactItems>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
             <plugin>
                 <groupId>org.wildfly.build</groupId>
                 <artifactId>wildfly-feature-pack-build-maven-plugin</artifactId>
+                <version>${wildfly.build-tools.version}</version>
                 <executions>
                     <execution>
                         <id>feature-pack-build</id>

distribution/feature-packs/server-feature-pack/src/main/resources/configuration/domain/template.xml

@@ -17,7 +17,7 @@
   ~ limitations under the License.
   -->
 
-<domain xmlns="urn:jboss:domain:5.0">
+<domain xmlns="urn:jboss:domain:7.0">
 
     <extensions>
         <?EXTENSIONS?>

distribution/feature-packs/server-feature-pack/src/main/resources/configuration/host/host-master.xml

@@ -22,7 +22,7 @@
   is also started by this host controller file.  The other instance must be started
   via host-slave.xml
 -->
-<host name="master" xmlns="urn:jboss:domain:5.0">
+<host name="master" xmlns="urn:jboss:domain:7.0">
     <extensions>
         <?EXTENSIONS?>
     </extensions>

distribution/feature-packs/server-feature-pack/src/main/resources/configuration/host/host-slave.xml

@@ -17,7 +17,7 @@
   ~ limitations under the License.
   -->
 
-<host xmlns="urn:jboss:domain:5.0">
+<host xmlns="urn:jboss:domain:7.0">
     <extensions>
         <?EXTENSIONS?>
     </extensions>

distribution/feature-packs/server-feature-pack/src/main/resources/configuration/host/host.xml

@@ -23,7 +23,7 @@
   via host-slave.xml
 -->
 
-<host name="master" xmlns="urn:jboss:domain:5.0">
+<host name="master" xmlns="urn:jboss:domain:7.0">
     <extensions>
         <?EXTENSIONS?>
     </extensions>

distribution/feature-packs/server-feature-pack/src/main/resources/configuration/standalone/template.xml

@@ -1,6 +1,6 @@
 <?xml version='1.0' encoding='UTF-8'?>
 
-<server xmlns="urn:jboss:domain:5.0">
+<server xmlns="urn:jboss:domain:7.0">
 
     <extensions>
         <?EXTENSIONS?>

distribution/feature-packs/server-feature-pack/src/main/resources/content/bin/migrate-domain-clustered.cli

@@ -115,7 +115,6 @@ if (result == undefined) of /profile=$clusteredProfile/subsystem=infinispan/cach
   /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/local-cache=keys/component=expiration/:write-attribute(name=max-idle,value=3600000)
   echo
 end-if
-
 if (outcome == failed) of /profile=$clusteredProfile/subsystem=keycloak-server/spi=publicKeyStorage/:read-resource
   echo Adding spi=publicKeyStorage...
   /profile=$clusteredProfile/subsystem=keycloak-server/spi=publicKeyStorage/:add
@@ -447,4 +446,98 @@ if (outcome == failed) of /profile=$clusteredProfile/subsystem=keycloak-server/s
   echo
 end-if
 
+# Migrate from 4.3.0 to 4.4.0
+if (outcome == failed) of /profile=$clusteredProfile/subsystem=elytron/permission-set=login-permission/:read-resource
+  echo Adding permission-set=login-permission to elytron
+  /profile=$clusteredProfile/subsystem=elytron/permission-set=login-permission:add(permissions=[{class-name=org.wildfly.security.auth.permission.LoginPermission}])
+  /profile=$clusteredProfile/subsystem=elytron/permission-set=default-permissions/:add(permissions=[{class-name=org.wildfly.extension.batch.jberet.deployment.BatchPermission,module=org.wildfly.extension.batch.jberet,target-name=*},{class-name=org.wildfly.transaction.client.RemoteTransactionPermission,module=org.wildfly.transaction.client},{class-name=org.jboss.ejb.client.RemoteEJBPermission,module=org.jboss.ejb-client}])
+  /profile=$clusteredProfile/subsystem=elytron/simple-permission-mapper=default-permission-mapper/:undefine-attribute(name=permission-mappings)
+  /profile=$clusteredProfile/subsystem=elytron/simple-permission-mapper=default-permission-mapper:write-attribute(name=permission-mappings,value=[{permission-sets=[{permission-set=login-permission},{permission-set=default-permissions}],match-all=true},{permission-sets=[{permission-set=default-permissions}],principals=[anonymous]}])
+  echo
+end-if
+
+if (result == org.hibernate.infinispan) of /profile=$clusteredProfile/subsystem=infinispan/cache-container=hibernate:read-attribute(name=module)
+  echo Update hibernate cache module
+  /profile=$clusteredProfile/subsystem=infinispan/cache-container=hibernate:write-attribute(name=module, value=org.infinispan.hibernate-cache)
+  echo
+end-if
+if (outcome == success) of /profile=$clusteredProfile/subsystem=infinispan/cache-container=hibernate:read-attribute(name=default-cache)
+  echo Remove default cache from hibernate cache
+  /profile=$clusteredProfile/subsystem=infinispan/cache-container=hibernate:undefine-attribute(name=default-cache)
+  echo
+end-if
+if (result == ASYNC) of /profile=$clusteredProfile/subsystem=infinispan/cache-container=hibernate/replicated-cache=timestamps:read-attribute(name=mode)
+  echo Switching mode for timestamps cache from ASYNC to SYNC
+  /profile=$clusteredProfile/subsystem=infinispan/cache-container=hibernate/replicated-cache=timestamps:write-attribute(name=mode, value=SYNC)
+  echo
+end-if
+
+if (outcome == success) of /profile=$clusteredProfile/subsystem=infinispan/cache-container=hibernate/local-cache=entity/eviction=EVICTION:read-resource
+  echo Removing eviction from hibernate entity cache and replacing with object-memory
+  /profile=$clusteredProfile/subsystem=infinispan/cache-container=hibernate/local-cache=entity/eviction=EVICTION:remove
+  /profile=$clusteredProfile/subsystem=infinispan/cache-container=hibernate/local-cache=entity/memory=object:add(size=10000)
+  echo
+end-if
+if (outcome == success) of /profile=$clusteredProfile/subsystem=infinispan/cache-container=hibernate/distributed-cache=local-query/eviction=EVICTION:read-resource
+  echo Removing eviction from hibernate local-query cache and replacing with object-memory
+  /profile=$clusteredProfile/subsystem=infinispan/cache-container=hibernate/local-cache=local-query/eviction=EVICTION:remove
+  /profile=$clusteredProfile/subsystem=infinispan/cache-container=hibernate/local-cache=local-query/memory=object:add(size=10000)
+  echo
+end-if
+if (outcome == success) of /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/local-cache=realms/eviction=EVICTION:read-resource
+  echo Removing eviction from keycloak realms cache and replacing with object-memory
+  /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/local-cache=realms/eviction=EVICTION:remove
+  /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/local-cache=realms/memory=object:add(size=10000)
+  echo
+end-if
+if (outcome == success) of /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/local-cache=users/eviction=EVICTION:read-resource
+  echo Removing eviction from keycloak users cache and replacing with object-memory
+  /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/local-cache=users/eviction=EVICTION:remove
+  /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/local-cache=users/memory=object:add(size=10000)
+  echo
+end-if
+if (outcome == success) of /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/local-cache=authorization/eviction=EVICTION:read-resource
+  echo Removing eviction from keycloak authorization cache and replacing with object-memory
+  /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/local-cache=authorization/eviction=EVICTION:remove
+  /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/local-cache=authorization/memory=object:add(size=10000)
+  echo
+end-if
+if (outcome == success) of /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/local-cache=keys/eviction=EVICTION:read-resource
+  echo Removing eviction from keycloak keys cache and replacing with object-memory
+  /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/local-cache=keys/eviction=EVICTION:remove
+  /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/local-cache=keys/memory=object:add(size=1000)
+  echo
+end-if
+
+if (outcome == success) of /profile=$clusteredProfile/subsystem=keycloak-server/spi=connectionsInfinispan/provider=default:read-resource
+  echo Changing JNDI reference in connectionsInfinispan SPI
+  /profile=$clusteredProfile/subsystem=keycloak-server/spi=connectionsInfinispan/provider=default:undefine-attribute(name=properties)
+  /profile=$clusteredProfile/subsystem=keycloak-server/spi=connectionsInfinispan/provider=default:write-attribute(name=properties,value={cacheContainer=java:jboss/infinispan/container/keycloak})
+  echo
+end-if
+
+if (outcome == success) of /profile=$clusteredProfile/subsystem=jgroups/stack=tcp/protocol=FRAG2:read-resource
+  echo Upgrade jgroups protocol from FRAG2 to FRAG3 for tcp stack
+  /profile=$clusteredProfile/subsystem=jgroups/stack=tcp/protocol=FRAG2:remove
+  /profile=$clusteredProfile/subsystem=jgroups/stack=tcp/protocol=FRAG3:add()
+  echo
+end-if
+if (outcome == success) of /profile=$clusteredProfile/subsystem=jgroups/stack=udp/protocol=FRAG2:read-resource
+  echo Upgrade jgroups protocol from FRAG2 to FRAG3 for udp stack
+  /profile=$clusteredProfile/subsystem=jgroups/stack=udp/protocol=FRAG2:remove
+  /profile=$clusteredProfile/subsystem=jgroups/stack=udp/protocol=FRAG3:add()
+  echo
+end-if
+if (outcome == success) of /profile=$clusteredProfile/subsystem=remoting/configuration=endpoint:read-resource
+  echo Remove endpoint from remoting configuration
+  /profile=$clusteredProfile/subsystem=remoting/configuration=endpoint:remove
+  echo
+end-if
+if (outcome == success) of /profile=$clusteredProfile/socket-binding-group=$clusteredProfile-sockets/socket-binding=jgroups-mping:read-attribute(name=port)
+  /profile=$clusteredProfile/socket-binding-group=$clusteredProfile-sockets/socket-binding=jgroups-mping:undefine-attribute(name=port)
+end-if
+if (outcome == success) of /socket-binding-group=$clusteredProfile-sockets/socket-binding=modcluster:read-attribute(name=port)
+  /profile=$clusteredProfile/socket-binding-group=$clusteredProfile-sockets/socket-binding=modcluster:undefine-attribute(name=port)
+end-if
+
 echo *** End Migration of /profile=$clusteredProfile ***

distribution/feature-packs/server-feature-pack/src/main/resources/content/bin/migrate-domain-standalone.cli

@@ -47,6 +47,7 @@ if (result == undefined) of /profile=$standaloneProfile/subsystem=infinispan/cac
   echo Updating authorization cache container..
   /profile=$standaloneProfile/subsystem=infinispan/cache-container=keycloak/local-cache=authorization/component=eviction/:write-attribute(name=strategy,value=LRU)
   /profile=$standaloneProfile/subsystem=infinispan/cache-container=keycloak/local-cache=authorization/component=eviction/:write-attribute(name=max-entries,value=100)
+  echo
 end-if
 
 # Migrate from 2.0.0 to 2.1.0
@@ -347,6 +348,7 @@ if (outcome == success) of /profile=$standaloneProfile/subsystem=undertow/server
   /profile=$standaloneProfile/subsystem=undertow/server=default-server/host=default-host/filter-ref=x-powered-by-header/:remove
   /profile=$standaloneProfile/subsystem=undertow/configuration=filter/response-header=x-powered-by-header/:remove
   /profile=$standaloneProfile/subsystem=undertow/configuration=filter/response-header=server-header/:remove
+  echo
 end-if
 
 if (outcome == success) of /profile=$standaloneProfile/subsystem=jdr/:read-resource
@@ -404,4 +406,63 @@ if (outcome == failed) of /profile=$standaloneProfile/subsystem=keycloak-server/
   echo
 end-if
 
+# Migrate from 4.3.0 to 4.4.0
+if (outcome == failed) of /profile=$standaloneProfile/subsystem=elytron/permission-set=login-permission/:read-resource
+  echo Adding permission-set=login-permission to elytron
+  /profile=$standaloneProfile/subsystem=elytron/permission-set=login-permission:add(permissions=[{class-name=org.wildfly.security.auth.permission.LoginPermission}])
+  /profile=$standaloneProfile/subsystem=elytron/permission-set=default-permissions/:add(permissions=[{class-name=org.wildfly.extension.batch.jberet.deployment.BatchPermission,module=org.wildfly.extension.batch.jberet,target-name=*},{class-name=org.wildfly.transaction.client.RemoteTransactionPermission,module=org.wildfly.transaction.client},{class-name=org.jboss.ejb.client.RemoteEJBPermission,module=org.jboss.ejb-client}])
+  /profile=$standaloneProfile/subsystem=elytron/simple-permission-mapper=default-permission-mapper/:undefine-attribute(name=permission-mappings)
+  /profile=$standaloneProfile/subsystem=elytron/simple-permission-mapper=default-permission-mapper:write-attribute(name=permission-mappings,value=[{permission-sets=[{permission-set=login-permission},{permission-set=default-permissions}],match-all=true},{permission-sets=[{permission-set=default-permissions}],principals=[anonymous]}])
+  echo
+end-if
+
+if (result == org.hibernate.infinispan) of /profile=$standaloneProfile/subsystem=infinispan/cache-container=hibernate:read-attribute(name=module)
+  echo Update hibernate cache module
+  /profile=$standaloneProfile/subsystem=infinispan/cache-container=hibernate:write-attribute(name=module, value=org.infinispan.hibernate-cache)
+  echo
+end-if
+if (outcome == success) of /profile=$standaloneProfile/subsystem=infinispan/cache-container=hibernate/local-cache=entity/eviction=EVICTION:read-resource
+  echo Removing eviction from hibernate entity cache and replacing with object-memory
+  /profile=$standaloneProfile/subsystem=infinispan/cache-container=hibernate/local-cache=entity/eviction=EVICTION:remove
+  /profile=$standaloneProfile/subsystem=infinispan/cache-container=hibernate/local-cache=entity/memory=object:add(size=10000)
+  echo
+end-if
+if (outcome == success) of /profile=$standaloneProfile/subsystem=infinispan/cache-container=hibernate/local-cache=local-query/eviction=EVICTION:read-resource
+  echo Removing eviction from hibernate local-query cache and replacing with object-memory
+  /profile=$standaloneProfile/subsystem=infinispan/cache-container=hibernate/local-cache=local-query/eviction=EVICTION:remove
+  /profile=$standaloneProfile/subsystem=infinispan/cache-container=hibernate/local-cache=local-query/memory=object:add(size=10000)
+  echo
+end-if
+if (outcome == success) of /profile=$standaloneProfile/subsystem=infinispan/cache-container=keycloak/local-cache=realms/eviction=EVICTION:read-resource
+  echo Removing eviction from keycloak realms cache and replacing with object-memory
+  /profile=$standaloneProfile/subsystem=infinispan/cache-container=keycloak/local-cache=realms/eviction=EVICTION:remove
+  /profile=$standaloneProfile/subsystem=infinispan/cache-container=keycloak/local-cache=realms/memory=object:add(size=10000)
+  echo
+end-if
+if (outcome == success) of /profile=$standaloneProfile/subsystem=infinispan/cache-container=keycloak/local-cache=users/eviction=EVICTION:read-resource
+  echo Removing eviction from keycloak users cache and replacing with object-memory
+  /profile=$standaloneProfile/subsystem=infinispan/cache-container=keycloak/local-cache=users/eviction=EVICTION:remove
+  /profile=$standaloneProfile/subsystem=infinispan/cache-container=keycloak/local-cache=users/memory=object:add(size=10000)
+  echo
+end-if
+if (outcome == success) of /profile=$standaloneProfile/subsystem=infinispan/cache-container=keycloak/local-cache=authorization/eviction=EVICTION:read-resource
+  echo Removing eviction from keycloak authorization cache and replacing with object-memory
+  /profile=$standaloneProfile/subsystem=infinispan/cache-container=keycloak/local-cache=authorization/eviction=EVICTION:remove
+  /profile=$standaloneProfile/subsystem=infinispan/cache-container=keycloak/local-cache=authorization/memory=object:add(size=10000)
+  echo
+end-if
+if (outcome == success) of /profile=$standaloneProfile/subsystem=infinispan/cache-container=keycloak/local-cache=keys/eviction=EVICTION:read-resource
+  echo Removing eviction from keycloak keys cache and replacing with object-memory
+  /profile=$standaloneProfile/subsystem=infinispan/cache-container=keycloak/local-cache=keys/eviction=EVICTION:remove
+  /profile=$standaloneProfile/subsystem=infinispan/cache-container=keycloak/local-cache=keys/memory=object:add(size=1000)
+  echo
+end-if
+
+if (outcome == success) of /profile=$standaloneProfile/subsystem=keycloak-server/spi=connectionsInfinispan/provider=default:read-resource
+  echo Changing JNDI reference in connectionsInfinispan SPI
+  /profile=$standaloneProfile/subsystem=keycloak-server/spi=connectionsInfinispan/provider=default:undefine-attribute(name=properties)
+  /profile=$standaloneProfile/subsystem=keycloak-server/spi=connectionsInfinispan/provider=default:write-attribute(name=properties,value={cacheContainer=java:jboss/infinispan/container/keycloak})
+  echo
+end-if
+
 echo *** End Migration of /profile=$standaloneProfile ***

distribution/feature-packs/server-feature-pack/src/main/resources/content/bin/migrate-standalone-ha.cli

@@ -137,7 +137,7 @@ if (outcome == success) of /subsystem=infinispan/cache-container=keycloak/invali
   echo
 end-if
 if (result == undefined) of /subsystem=infinispan/cache-container=keycloak/local-cache=users/component=eviction/:read-attribute(name=strategy,include-defaults=false)
-  echo Updating eviction in local-cache=users...
+  echo Updating eviction in local-cache=users
   /subsystem=infinispan/cache-container=keycloak/local-cache=users/component=eviction/:write-attribute(name=strategy,value=LRU)
   /subsystem=infinispan/cache-container=keycloak/local-cache=users/component=eviction/:write-attribute(name=max-entries,value=10000)
   echo
@@ -431,4 +431,100 @@ if (outcome == failed) of /subsystem=keycloak-server/spi=hostname/:read-resource
   echo
 end-if
 
+# Migrate from 4.3.0 to 4.4.0
+if (outcome == failed) of /subsystem=elytron/permission-set=login-permission/:read-resource
+  echo Adding permission-set=login-permission to elytron
+  /subsystem=elytron/permission-set=login-permission:add(permissions=[{class-name=org.wildfly.security.auth.permission.LoginPermission}])
+  /subsystem=elytron/permission-set=default-permissions/:add(permissions=[{class-name=org.wildfly.extension.batch.jberet.deployment.BatchPermission,module=org.wildfly.extension.batch.jberet,target-name=*},{class-name=org.wildfly.transaction.client.RemoteTransactionPermission,module=org.wildfly.transaction.client},{class-name=org.jboss.ejb.client.RemoteEJBPermission,module=org.jboss.ejb-client}])
+  /subsystem=elytron/simple-permission-mapper=default-permission-mapper/:undefine-attribute(name=permission-mappings)
+  /subsystem=elytron/simple-permission-mapper=default-permission-mapper:write-attribute(name=permission-mappings,value=[{permission-sets=[{permission-set=login-permission},{permission-set=default-permissions}],match-all=true},{permission-sets=[{permission-set=default-permissions}],principals=[anonymous]}])
+  echo
+end-if
+
+
+if (result == org.hibernate.infinispan) of /subsystem=infinispan/cache-container=hibernate:read-attribute(name=module)
+  echo Update hibernate cache module
+  /subsystem=infinispan/cache-container=hibernate:write-attribute(name=module, value=org.infinispan.hibernate-cache)
+  echo
+end-if
+if (outcome == success) of /subsystem=infinispan/cache-container=hibernate:read-attribute(name=default-cache)
+  echo Remove default cache from hibernate cache
+  /subsystem=infinispan/cache-container=hibernate:undefine-attribute(name=default-cache)
+  echo
+end-if
+if (result == ASYNC) of /subsystem=infinispan/cache-container=hibernate/replicated-cache=timestamps:read-attribute(name=mode)
+  echo Switching mode for timestamps cache from ASYNC to SYNC
+  /subsystem=infinispan/cache-container=hibernate/replicated-cache=timestamps:write-attribute(name=mode, value=SYNC)
+  echo
+end-if
+
+if (outcome == success) of /subsystem=infinispan/cache-container=hibernate/local-cache=entity/eviction=EVICTION:read-resource
+  echo Removing eviction from hibernate entity cache and replacing with object-memory
+  /subsystem=infinispan/cache-container=hibernate/local-cache=entity/eviction=EVICTION:remove
+  /subsystem=infinispan/cache-container=hibernate/local-cache=entity/memory=object:add(size=10000)
+  echo
+end-if
+if (outcome == success) of /subsystem=infinispan/cache-container=hibernate/distributed-cache=local-query/eviction=EVICTION:read-resource
+  echo Removing eviction from hibernate local-query cache and replacing with object-memory
+  /subsystem=infinispan/cache-container=hibernate/local-cache=local-query/eviction=EVICTION:remove
+  /subsystem=infinispan/cache-container=hibernate/local-cache=local-query/memory=object:add(size=10000)
+  echo
+end-if
+
+if (outcome == success) of /subsystem=infinispan/cache-container=keycloak/local-cache=realms/eviction=EVICTION:read-resource
+  echo Removing eviction from keycloak realms cache and replacing with object-memory
+  /subsystem=infinispan/cache-container=keycloak/local-cache=realms/eviction=EVICTION:remove
+  /subsystem=infinispan/cache-container=keycloak/local-cache=realms/memory=object:add(size=10000)
+  echo
+end-if
+if (outcome == success) of /subsystem=infinispan/cache-container=keycloak/local-cache=users/eviction=EVICTION:read-resource
+  echo Removing eviction from keycloak users cache and replacing with object-memory
+  /subsystem=infinispan/cache-container=keycloak/local-cache=users/eviction=EVICTION:remove
+  /subsystem=infinispan/cache-container=keycloak/local-cache=users/memory=object:add(size=10000)
+  echo
+end-if
+if (outcome == success) of /subsystem=infinispan/cache-container=keycloak/local-cache=authorization/eviction=EVICTION:read-resource
+  echo Removing eviction from keycloak authorization cache and replacing with object-memory
+  /subsystem=infinispan/cache-container=keycloak/local-cache=authorization/eviction=EVICTION:remove
+  /subsystem=infinispan/cache-container=keycloak/local-cache=authorization/memory=object:add(size=10000)
+  echo
+end-if
+if (outcome == success) of /subsystem=infinispan/cache-container=keycloak/local-cache=keys/eviction=EVICTION:read-resource
+  echo Removing eviction from keycloak keys cache and replacing with object-memory
+  /subsystem=infinispan/cache-container=keycloak/local-cache=keys/eviction=EVICTION:remove
+  /subsystem=infinispan/cache-container=keycloak/local-cache=keys/memory=object:add(size=1000)
+  echo
+end-if
+
+if (outcome == success) of /subsystem=jgroups/stack=tcp/protocol=FRAG2:read-resource
+  echo Upgrade jgroups protocol from FRAG2 to FRAG3 for tcp stack
+  /subsystem=jgroups/stack=tcp/protocol=FRAG2:remove
+  /subsystem=jgroups/stack=tcp/protocol=FRAG3:add()
+  echo
+end-if
+if (outcome == success) of /subsystem=jgroups/stack=udp/protocol=FRAG2:read-resource
+  echo Upgrade jgroups protocol from FRAG2 to FRAG3 for udp stack
+  /subsystem=jgroups/stack=udp/protocol=FRAG2:remove
+  /subsystem=jgroups/stack=udp/protocol=FRAG3:add()
+  echo
+end-if
+if (outcome == success) of /subsystem=remoting/configuration=endpoint:read-resource
+  echo Remove endpoint from remoting configuration
+  /subsystem=remoting/configuration=endpoint:remove
+  echo
+end-if
+if (outcome == success) of /socket-binding-group=standard-sockets/socket-binding=jgroups-mping:read-attribute(name=port)
+  /socket-binding-group=standard-sockets/socket-binding=jgroups-mping:undefine-attribute(name=port)
+end-if
+if (outcome == success) of /socket-binding-group=standard-sockets/socket-binding=modcluster:read-attribute(name=port)
+  /socket-binding-group=standard-sockets/socket-binding=modcluster:undefine-attribute(name=port)
+end-if
+
+if (outcome == success) of /subsystem=keycloak-server/spi=connectionsInfinispan/provider=default:read-resource
+  echo Changing JNDI reference in connectionsInfinispan SPI
+  /subsystem=keycloak-server/spi=connectionsInfinispan/provider=default:undefine-attribute(name=properties)
+  /subsystem=keycloak-server/spi=connectionsInfinispan/provider=default:write-attribute(name=properties,value={cacheContainer=java:jboss/infinispan/container/keycloak})
+  echo
+end-if
+
 echo *** End Migration ***

distribution/feature-packs/server-feature-pack/src/main/resources/content/bin/migrate-standalone.cli

@@ -375,4 +375,63 @@ if (outcome == failed) of /subsystem=keycloak-server/spi=hostname/:read-resource
   echo
 end-if
 
+# Migrate from 4.3.0 to 4.4.0
+if (outcome == failed) of /subsystem=elytron/permission-set=login-permission/:read-resource
+  echo Adding permission-set=login-permission to elytron
+  /subsystem=elytron/permission-set=login-permission:add(permissions=[{class-name=org.wildfly.security.auth.permission.LoginPermission}])
+  /subsystem=elytron/permission-set=default-permissions/:add(permissions=[{class-name=org.wildfly.extension.batch.jberet.deployment.BatchPermission,module=org.wildfly.extension.batch.jberet,target-name=*},{class-name=org.wildfly.transaction.client.RemoteTransactionPermission,module=org.wildfly.transaction.client},{class-name=org.jboss.ejb.client.RemoteEJBPermission,module=org.jboss.ejb-client}])
+  /subsystem=elytron/simple-permission-mapper=default-permission-mapper/:undefine-attribute(name=permission-mappings)
+  /subsystem=elytron/simple-permission-mapper=default-permission-mapper:write-attribute(name=permission-mappings,value=[{permission-sets=[{permission-set=login-permission},{permission-set=default-permissions}],match-all=true},{permission-sets=[{permission-set=default-permissions}],principals=[anonymous]}])
+  echo
+end-if
+
+if (result == org.hibernate.infinispan) of /subsystem=infinispan/cache-container=hibernate:read-attribute(name=module)
+  echo Update hibernate cache module
+  /subsystem=infinispan/cache-container=hibernate:write-attribute(name=module, value=org.infinispan.hibernate-cache)
+  echo
+end-if
+if (outcome == success) of /subsystem=infinispan/cache-container=hibernate/local-cache=entity/eviction=EVICTION:read-resource
+  echo Removing eviction from hibernate entity cache and replacing with object-memory
+  /subsystem=infinispan/cache-container=hibernate/local-cache=entity/eviction=EVICTION:remove
+  /subsystem=infinispan/cache-container=hibernate/local-cache=entity/memory=object:add(size=10000)
+  echo
+end-if
+if (outcome == success) of /subsystem=infinispan/cache-container=hibernate/local-cache=local-query/eviction=EVICTION:read-resource
+  echo Removing eviction from hibernate local-query cache and replacing with object-memory
+  /subsystem=infinispan/cache-container=hibernate/local-cache=local-query/eviction=EVICTION:remove
+  /subsystem=infinispan/cache-container=hibernate/local-cache=local-query/memory=object:add(size=10000)
+  echo
+end-if
+if (outcome == success) of /subsystem=infinispan/cache-container=keycloak/local-cache=realms/eviction=EVICTION:read-resource
+  echo Removing eviction from keycloak realms cache and replacing with object-memory
+  /subsystem=infinispan/cache-container=keycloak/local-cache=realms/eviction=EVICTION:remove
+  /subsystem=infinispan/cache-container=keycloak/local-cache=realms/memory=object:add(size=10000)
+  echo
+end-if
+if (outcome == success) of /subsystem=infinispan/cache-container=keycloak/local-cache=users/eviction=EVICTION:read-resource
+  echo Removing eviction from keycloak users cache and replacing with object-memory
+  /subsystem=infinispan/cache-container=keycloak/local-cache=users/eviction=EVICTION:remove
+  /subsystem=infinispan/cache-container=keycloak/local-cache=users/memory=object:add(size=10000)
+  echo
+end-if
+if (outcome == success) of /subsystem=infinispan/cache-container=keycloak/local-cache=authorization/eviction=EVICTION:read-resource
+  echo Removing eviction from keycloak authorization cache and replacing with object-memory
+  /subsystem=infinispan/cache-container=keycloak/local-cache=authorization/eviction=EVICTION:remove
+  /subsystem=infinispan/cache-container=keycloak/local-cache=authorization/memory=object:add(size=10000)
+  echo
+end-if
+if (outcome == success) of /subsystem=infinispan/cache-container=keycloak/local-cache=keys/eviction=EVICTION:read-resource
+  echo Removing eviction from keycloak keys cache and replacing with object-memory
+  /subsystem=infinispan/cache-container=keycloak/local-cache=keys/eviction=EVICTION:remove
+  /subsystem=infinispan/cache-container=keycloak/local-cache=keys/memory=object:add(size=1000)
+  echo
+end-if
+
+if (outcome == success) of /subsystem=keycloak-server/spi=connectionsInfinispan/provider=default:read-resource
+  echo Changing JNDI reference in connectionsInfinispan SPI
+  /subsystem=keycloak-server/spi=connectionsInfinispan/provider=default:undefine-attribute(name=properties)
+  /subsystem=keycloak-server/spi=connectionsInfinispan/provider=default:write-attribute(name=properties,value={cacheContainer=java:jboss/infinispan/container/keycloak})
+  echo
+end-if
+
 echo *** End Migration ***

distribution/feature-packs/server-feature-pack/src/main/resources/modules/system/layers/keycloak/org/keycloak/keycloak-model-infinispan/main/module.xml

@@ -31,10 +31,11 @@
         <module name="org.keycloak.keycloak-server-spi-private"/>
         <module name="org.infinispan"/>
         <module name="org.infinispan.commons"/>
-        <module name="org.infinispan.cachestore.remote"/>
+        <module name="org.infinispan.persistence.remote"/>
         <module name="org.infinispan.client.hotrod"/>
         <module name="org.jgroups"/>
         <module name="org.jboss.logging"/>
+        <module name="io.netty"/>
         <module name="javax.api"/>
     </dependencies>
 </module>

distribution/feature-packs/server-feature-pack/src/main/resources/modules/system/layers/keycloak/org/keycloak/keycloak-server-subsystem/main/server-war/WEB-INF/web.xml

@@ -61,6 +61,6 @@
     <resource-env-ref>
         <resource-env-ref-name>infinispan/Keycloak</resource-env-ref-name>
         <resource-env-ref-type>org.infinispan.manager.EmbeddedCacheManager</resource-env-ref-type>
-        <lookup-name>java:jboss/infinispan/Keycloak</lookup-name>
+        <lookup-name>java:jboss/infinispan/container/keycloak</lookup-name>
     </resource-env-ref>
 </web-app>

distribution/feature-packs/server-feature-pack/src/main/resources/modules/system/layers/keycloak/org/keycloak/keycloak-wildfly-adduser/main/module.xml

@@ -15,7 +15,7 @@
   ~ See the License for the specific language governing permissions and
   ~ limitations under the License.
   -->
-<module xmlns="urn:jboss:module:1.3" name="org.keycloak.keycloak-wildfly-adduser">
+<module xmlns="urn:jboss:module:1.6" name="org.keycloak.keycloak-wildfly-adduser">
     <main-class name="org.keycloak.wildfly.adduser.AddUser"/>
 
     <properties>
@@ -32,7 +32,7 @@
         <module name="org.keycloak.keycloak-server-spi" services="import"/>
         <module name="org.keycloak.keycloak-server-spi-private" services="import"/>
         <module name="org.keycloak.keycloak-services" services="import"/>
-        <module name="org.jboss.aesh"/>
+        <module name="org.aesh"/>
         <module name="org.jboss.as.domain-management"/>
         <module name="com.fasterxml.jackson.core.jackson-core"/>
         <module name="com.fasterxml.jackson.core.jackson-annotations"/>

distribution/server-dist/assembly.xml

@@ -102,14 +102,6 @@
                 <include>layers.conf</include>
             </includes>
         </fileSet>
-        <fileSet>
-            <directory>target/unpacked/keycloak-client-tools</directory>
-            <outputDirectory/>
-            <filtered>false</filtered>
-            <includes>
-                <include>**/*</include>
-            </includes>
-        </fileSet>
         <fileSet>
             <directory>target/licenses/content/docs</directory>
             <outputDirectory>docs</outputDirectory>

distribution/server-dist/pom.xml

@@ -41,17 +41,6 @@
                 </exclusion>
             </exclusions>
         </dependency>
-        <dependency>
-            <groupId>org.keycloak</groupId>
-            <artifactId>keycloak-client-cli-dist</artifactId>
-            <type>zip</type>
-            <exclusions>
-                <exclusion>
-                    <groupId>*</groupId>
-                    <artifactId>*</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
     </dependencies>
 
     <build>

distribution/server-overlay/src/main/cli/keycloak-install-base.cli

@@ -1,10 +1,9 @@
 embed-server --server-config=standalone.xml
 /subsystem=datasources/data-source=KeycloakDS/:add(connection-url="jdbc:h2:${jboss.server.data.dir}/keycloak;AUTO_SERVER=TRUE",enabled=true,driver-name=h2,jndi-name=java:jboss/datasources/KeycloakDS,password=sa,user-name=sa,use-java-context=true)
-/subsystem=infinispan/cache-container=keycloak:add(jndi-name="infinispan/Keycloak")
 /subsystem=infinispan/cache-container=keycloak/local-cache=realms:add()
-/subsystem=infinispan/cache-container=keycloak/local-cache=realms/eviction=EVICTION:add(max-entries=10000,strategy=LRU)
+/subsystem=infinispan/cache-container=keycloak/local-cache=realms/memory=object:add(size=10000)
 /subsystem=infinispan/cache-container=keycloak/local-cache=users:add()
-/subsystem=infinispan/cache-container=keycloak/local-cache=users/eviction=EVICTION:add(max-entries=10000,strategy=LRU)
+/subsystem=infinispan/cache-container=keycloak/local-cache=users/memory=object:add(size=10000)
 /subsystem=infinispan/cache-container=keycloak/local-cache=sessions:add()
 /subsystem=infinispan/cache-container=keycloak/local-cache=authenticationSessions:add()
 /subsystem=infinispan/cache-container=keycloak/local-cache=offlineSessions:add()
@@ -13,11 +12,11 @@ embed-server --server-config=standalone.xml
 /subsystem=infinispan/cache-container=keycloak/local-cache=loginFailures:add()
 /subsystem=infinispan/cache-container=keycloak/local-cache=work:add()
 /subsystem=infinispan/cache-container=keycloak/local-cache=authorization:add()
-/subsystem=infinispan/cache-container=keycloak/local-cache=authorization/eviction=EVICTION:add(max-entries=100,strategy=LRU)
+/subsystem=infinispan/cache-container=keycloak/local-cache=authorization/memory=object:add(size=10000)
 /subsystem=infinispan/cache-container=keycloak/local-cache=keys:add()
-/subsystem=infinispan/cache-container=keycloak/local-cache=keys/eviction=EVICTION:add(max-entries=1000,strategy=LRU)
+/subsystem=infinispan/cache-container=keycloak/local-cache=keys/memory=object:add(size=1000)
 /subsystem=infinispan/cache-container=keycloak/local-cache=keys/expiration=EXPIRATION:add(max-idle=3600000)
 /subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens:add()
-/subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/eviction=EVICTION:add(max-entries=-1,strategy=NONE)
+/subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/memory=object:add(size=-1)
 /subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/expiration=EXPIRATION:add(max-idle=-1,interval=300000)
 /extension=org.keycloak.keycloak-server-subsystem/:add(module=org.keycloak.keycloak-server-subsystem)

distribution/server-overlay/src/main/cli/keycloak-install-ha-base.cli

@@ -1,24 +1,23 @@
 embed-server --server-config=standalone-ha.xml
 /subsystem=datasources/data-source=KeycloakDS/:add(connection-url="jdbc:h2:${jboss.server.data.dir}/keycloak;AUTO_SERVER=TRUE",enabled=true,driver-name=h2,jndi-name=java:jboss/datasources/KeycloakDS,password=sa,user-name=sa,use-java-context=true)
-/subsystem=infinispan/cache-container=keycloak:add(jndi-name="infinispan/Keycloak")
 /subsystem=infinispan/cache-container=keycloak/transport=TRANSPORT:add(lock-timeout=60000)
 /subsystem=infinispan/cache-container=keycloak/local-cache=realms:add()
-/subsystem=infinispan/cache-container=keycloak/local-cache=realms/eviction=EVICTION:add(max-entries=10000,strategy=LRU)
+/subsystem=infinispan/cache-container=keycloak/local-cache=realms/memory=object:add(size=10000)
 /subsystem=infinispan/cache-container=keycloak/local-cache=users:add()
-/subsystem=infinispan/cache-container=keycloak/local-cache=users/eviction=EVICTION:add(max-entries=10000,strategy=LRU)
+/subsystem=infinispan/cache-container=keycloak/local-cache=users/memory=object:add(size=10000)
-/subsystem=infinispan/cache-container=keycloak/distributed-cache=sessions:add(mode="SYNC",owners="1")
+/subsystem=infinispan/cache-container=keycloak/distributed-cache=sessions:add(owners="1")
-/subsystem=infinispan/cache-container=keycloak/distributed-cache=authenticationSessions:add(mode="SYNC",owners="1")
+/subsystem=infinispan/cache-container=keycloak/distributed-cache=authenticationSessions:add(owners="1")
-/subsystem=infinispan/cache-container=keycloak/distributed-cache=offlineSessions:add(mode="SYNC",owners="1")
+/subsystem=infinispan/cache-container=keycloak/distributed-cache=offlineSessions:add(owners="1")
-/subsystem=infinispan/cache-container=keycloak/distributed-cache=clientSessions:add(mode="SYNC",owners="1")
+/subsystem=infinispan/cache-container=keycloak/distributed-cache=clientSessions:add(owners="1")
-/subsystem=infinispan/cache-container=keycloak/distributed-cache=offlineClientSessions:add(mode="SYNC",owners="1")
+/subsystem=infinispan/cache-container=keycloak/distributed-cache=offlineClientSessions:add(owners="1")
-/subsystem=infinispan/cache-container=keycloak/distributed-cache=loginFailures:add(mode="SYNC",owners="1")
+/subsystem=infinispan/cache-container=keycloak/distributed-cache=loginFailures:add(owners="1")
 /subsystem=infinispan/cache-container=keycloak/local-cache=authorization:add()
-/subsystem=infinispan/cache-container=keycloak/local-cache=authorization/eviction=EVICTION:add(max-entries=10000,strategy=LRU)
+/subsystem=infinispan/cache-container=keycloak/local-cache=authorization/memory=object:add(size=10000)
-/subsystem=infinispan/cache-container=keycloak/replicated-cache=work:add(mode="SYNC")
+/subsystem=infinispan/cache-container=keycloak/replicated-cache=work:add()
 /subsystem=infinispan/cache-container=keycloak/local-cache=keys:add()
-/subsystem=infinispan/cache-container=keycloak/local-cache=keys/eviction=EVICTION:add(max-entries=1000,strategy=LRU)
+/subsystem=infinispan/cache-container=keycloak/local-cache=keys/memory=object:add(size=1000)
 /subsystem=infinispan/cache-container=keycloak/local-cache=keys/expiration=EXPIRATION:add(max-idle=3600000)
-/subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens:add(indexing="NONE",mode="SYNC",owners="2")
+/subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens:add(indexing="NONE",owners="2")
-/subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/eviction=EVICTION:add(max-entries=-1,strategy=NONE)
+/subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/memory=object:add(size=-1)
 /subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/expiration=EXPIRATION:add(max-idle=-1,interval=300000)
 /extension=org.keycloak.keycloak-server-subsystem/:add(module=org.keycloak.keycloak-server-subsystem)

distribution/server-provisioning-devel.xml

@@ -15,9 +15,6 @@
   ~ limitations under the License.
   -->
 <server-provisioning xmlns="urn:wildfly:server-provisioning:1.2" extract-schemas="true">
-    <copy-artifacts>
-        <copy-artifact artifact="org.keycloak:keycloak-client-cli-dist:zip" to-location="" from-location="keycloak-client-tools"/>
-    </copy-artifacts>
     <feature-packs>
         <feature-pack groupId="org.keycloak" artifactId="keycloak-server-feature-pack" version="${project.version}"/>
     </feature-packs>

distribution/server-provisioning.xml

@@ -15,9 +15,6 @@
   ~ limitations under the License.
   -->
 <server-provisioning xmlns="urn:wildfly:server-provisioning:1.2" extract-schemas="true" copy-module-artifacts="true">
-    <copy-artifacts>
-        <copy-artifact artifact="org.keycloak:keycloak-client-cli-dist:zip" to-location="" from-location="keycloak-client-tools"/>
-    </copy-artifacts>
     <feature-packs>
         <feature-pack groupId="org.keycloak" artifactId="keycloak-server-feature-pack" version="${project.version}"/>
     </feature-packs>

examples/broker/twitter-authentication/pom.xml

@@ -66,7 +66,7 @@
         </dependency>
         <dependency>
             <groupId>org.jboss.spec.javax.ws.rs</groupId>
-            <artifactId>jboss-jaxrs-api_2.0_spec</artifactId>
+            <artifactId>jboss-jaxrs-api_2.1_spec</artifactId>
             <scope>provided</scope>
         </dependency>
     </dependencies>

examples/providers/domain-extension/pom.xml

@@ -58,7 +58,7 @@
         </dependency>
         <dependency>
             <groupId>org.jboss.spec.javax.ws.rs</groupId>
-            <artifactId>jboss-jaxrs-api_2.0_spec</artifactId>
+            <artifactId>jboss-jaxrs-api_2.1_spec</artifactId>
         </dependency>
     </dependencies>
 

examples/providers/rest/pom.xml

@@ -48,7 +48,7 @@
         </dependency>
         <dependency>
             <groupId>org.jboss.spec.javax.ws.rs</groupId>
-            <artifactId>jboss-jaxrs-api_2.0_spec</artifactId>
+            <artifactId>jboss-jaxrs-api_2.1_spec</artifactId>
         </dependency>
     </dependencies>
 

integration/admin-client/pom.xml

@@ -47,7 +47,7 @@
         </dependency>
         <dependency>
             <groupId>org.jboss.spec.javax.ws.rs</groupId>
-            <artifactId>jboss-jaxrs-api_2.0_spec</artifactId>
+            <artifactId>jboss-jaxrs-api_2.1_spec</artifactId>
             <scope>provided</scope>
         </dependency>
         <dependency>

integration/client-cli/admin-cli/pom.xml

@@ -145,7 +145,7 @@
                                     </includes>
                                 </filter>
                                 <filter>
-                                    <artifact>org.jboss.spec.javax.ws.rs:jboss-jaxrs-api_2.0_spec</artifact>
+                                    <artifact>org.jboss.spec.javax.ws.rs:jboss-jaxrs-api_2.1_spec</artifact>
                                     <includes>
                                         <include>**/**</include>
                                     </includes>

model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanClusterProviderFactory.java

@@ -33,6 +33,7 @@ import org.keycloak.cluster.ClusterProviderFactory;
 import org.keycloak.common.util.Retry;
 import org.keycloak.common.util.Time;
 import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
+import org.keycloak.connections.infinispan.TopologyInfo;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
 import org.keycloak.models.sessions.infinispan.util.InfinispanUtil;
@@ -77,7 +78,7 @@ public class InfinispanClusterProviderFactory implements ClusterProviderFactory
     @Override
     public ClusterProvider create(KeycloakSession session) {
         lazyInit(session);
-        String myAddress = InfinispanUtil.getMyAddress(session);
+        String myAddress = InfinispanUtil.getTopologyInfo(session).getMyNodeName();
         return new InfinispanClusterProvider(clusterStartupTime, myAddress, crossDCAwareCacheFactory, notificationsManager, localExecutor);
     }
 
@@ -96,8 +97,9 @@ public class InfinispanClusterProviderFactory implements ClusterProviderFactory
 
                     clusterStartupTime = initClusterStartupTime(session);
 
-                    String myAddress = InfinispanUtil.getMyAddress(session);
+                    TopologyInfo topologyInfo = InfinispanUtil.getTopologyInfo(session);
-                    String mySite = InfinispanUtil.getMySite(session);
+                    String myAddress = topologyInfo.getMyNodeName();
+                    String mySite = topologyInfo.getMySiteName();
 
                     notificationsManager = InfinispanNotificationsManager.create(session, workCache, myAddress, mySite, remoteStores);
                 }

model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProvider.java

@@ -28,14 +28,12 @@ public class DefaultInfinispanConnectionProvider implements InfinispanConnection
 
     private final EmbeddedCacheManager cacheManager;
     private final RemoteCacheProvider remoteCacheProvider;
-    private final String siteName;
+    private final TopologyInfo topologyInfo;
-    private final String nodeName;
 
-    public DefaultInfinispanConnectionProvider(EmbeddedCacheManager cacheManager, RemoteCacheProvider remoteCacheProvider, String nodeName, String siteName) {
+    public DefaultInfinispanConnectionProvider(EmbeddedCacheManager cacheManager, RemoteCacheProvider remoteCacheProvider, TopologyInfo topologyInfo) {
         this.cacheManager = cacheManager;
         this.remoteCacheProvider = remoteCacheProvider;
-        this.nodeName = nodeName;
+        this.topologyInfo = topologyInfo;
-        this.siteName = siteName;
     }
 
     @Override
@@ -49,13 +47,8 @@ public class DefaultInfinispanConnectionProvider implements InfinispanConnection
     }
 
     @Override
-    public String getNodeName() {
+    public TopologyInfo getTopologyInfo() {
-        return nodeName;
+        return topologyInfo;
-    }
-
-    @Override
-    public String getSiteName() {
-        return siteName;
     }
 
     @Override

model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java

@@ -20,6 +20,7 @@ package org.keycloak.connections.infinispan;
 import java.security.SecureRandom;
 import java.util.concurrent.TimeUnit;
 
+import org.infinispan.client.hotrod.ProtocolVersion;
 import org.infinispan.commons.util.FileLookup;
 import org.infinispan.commons.util.FileLookupFactory;
 import org.infinispan.configuration.cache.CacheMode;
@@ -34,7 +35,7 @@ import org.infinispan.remoting.transport.Transport;
 import org.infinispan.remoting.transport.jgroups.JGroupsTransport;
 import org.infinispan.transaction.LockingMode;
 import org.infinispan.transaction.TransactionMode;
-import org.infinispan.transaction.lookup.DummyTransactionManagerLookup;
+import org.infinispan.transaction.lookup.EmbeddedTransactionManagerLookup;
 import org.jboss.logging.Logger;
 import org.jgroups.JChannel;
 import org.keycloak.Config;
@@ -60,15 +61,13 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
 
     protected boolean containerManaged;
 
-    private String nodeName;
+    private TopologyInfo topologyInfo;
-
-    private String siteName;
 
     @Override
     public InfinispanConnectionProvider create(KeycloakSession session) {
         lazyInit();
 
-        return new DefaultInfinispanConnectionProvider(cacheManager, remoteCacheProvider, nodeName, siteName);
+        return new DefaultInfinispanConnectionProvider(cacheManager, remoteCacheProvider, topologyInfo);
     }
 
     @Override
@@ -108,7 +107,7 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
                         initEmbedded();
                     }
 
-                    logger.infof("Node name: %s, Site name: %s", nodeName, siteName);
+                    logger.infof(topologyInfo.toString());
 
                     remoteCacheProvider = new RemoteCacheProvider(config, cacheManager);
                 }
@@ -121,7 +120,7 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
             cacheManager = (EmbeddedCacheManager) new InitialContext().lookup(cacheContainerLookup);
             containerManaged = true;
 
-            long realmRevisionsMaxEntries = cacheManager.getCache(InfinispanConnectionProvider.REALM_CACHE_NAME).getCacheConfiguration().eviction().maxEntries();
+            long realmRevisionsMaxEntries = cacheManager.getCache(InfinispanConnectionProvider.REALM_CACHE_NAME).getCacheConfiguration().memory().size();
             realmRevisionsMaxEntries = realmRevisionsMaxEntries > 0
                     ? 2 * realmRevisionsMaxEntries
                     : InfinispanConnectionProvider.REALM_REVISIONS_CACHE_DEFAULT_MAX;
@@ -129,7 +128,7 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
             cacheManager.defineConfiguration(InfinispanConnectionProvider.REALM_REVISIONS_CACHE_NAME, getRevisionCacheConfig(realmRevisionsMaxEntries));
             cacheManager.getCache(InfinispanConnectionProvider.REALM_REVISIONS_CACHE_NAME, true);
 
-            long userRevisionsMaxEntries = cacheManager.getCache(InfinispanConnectionProvider.USER_CACHE_NAME).getCacheConfiguration().eviction().maxEntries();
+            long userRevisionsMaxEntries = cacheManager.getCache(InfinispanConnectionProvider.USER_CACHE_NAME).getCacheConfiguration().memory().size();
             userRevisionsMaxEntries = userRevisionsMaxEntries > 0
                     ? 2 * userRevisionsMaxEntries
                     : InfinispanConnectionProvider.USER_REVISIONS_CACHE_DEFAULT_MAX;
@@ -141,7 +140,7 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
             cacheManager.getCache(InfinispanConnectionProvider.KEYS_CACHE_NAME, true);
             cacheManager.getCache(InfinispanConnectionProvider.ACTION_TOKEN_CACHE, true);
 
-            long authzRevisionsMaxEntries = cacheManager.getCache(InfinispanConnectionProvider.AUTHORIZATION_CACHE_NAME).getCacheConfiguration().eviction().maxEntries();
+            long authzRevisionsMaxEntries = cacheManager.getCache(InfinispanConnectionProvider.AUTHORIZATION_CACHE_NAME).getCacheConfiguration().memory().size();
             authzRevisionsMaxEntries = authzRevisionsMaxEntries > 0
                     ? 2 * authzRevisionsMaxEntries
                     : InfinispanConnectionProvider.AUTHORIZATION_REVISIONS_CACHE_DEFAULT_MAX;
@@ -149,20 +148,7 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
             cacheManager.defineConfiguration(InfinispanConnectionProvider.AUTHORIZATION_REVISIONS_CACHE_NAME, getRevisionCacheConfig(authzRevisionsMaxEntries));
             cacheManager.getCache(InfinispanConnectionProvider.AUTHORIZATION_REVISIONS_CACHE_NAME, true);
 
-            Transport transport = cacheManager.getTransport();
+            this.topologyInfo = new TopologyInfo(cacheManager, config, false);
-            if (transport != null) {
-                this.nodeName = transport.getAddress().toString();
-                this.siteName = cacheManager.getCacheManagerConfiguration().transport().siteId();
-                if (this.siteName == null) {
-                    this.siteName = System.getProperty(InfinispanConnectionProvider.JBOSS_SITE_NAME);
-                }
-            } else {
-                this.nodeName = System.getProperty(InfinispanConnectionProvider.JBOSS_NODE_NAME);
-                this.siteName = System.getProperty(InfinispanConnectionProvider.JBOSS_SITE_NAME);
-            }
-            if (this.nodeName == null || this.nodeName.equals("localhost")) {
-                this.nodeName = generateNodeName();
-            }
 
             logger.debugv("Using container managed Infinispan cache container, lookup={0}", cacheContainerLookup);
         } catch (Exception e) {
@@ -180,25 +166,13 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
         boolean async = config.getBoolean("async", false);
         boolean allowDuplicateJMXDomains = config.getBoolean("allowDuplicateJMXDomains", true);
 
-        this.nodeName = config.get("nodeName", System.getProperty(InfinispanConnectionProvider.JBOSS_NODE_NAME));
+        this.topologyInfo = new TopologyInfo(cacheManager, config, true);
-        if (this.nodeName != null && this.nodeName.isEmpty()) {
-            this.nodeName = null;
-        }
-
-        this.siteName = config.get("siteName", System.getProperty(InfinispanConnectionProvider.JBOSS_SITE_NAME));
-        if (this.siteName != null && this.siteName.isEmpty()) {
-            this.siteName = null;
-        }
 
         if (clustered) {
             String jgroupsUdpMcastAddr = config.get("jgroupsUdpMcastAddr", System.getProperty(InfinispanConnectionProvider.JGROUPS_UDP_MCAST_ADDR));
-            configureTransport(gcb, nodeName, siteName, jgroupsUdpMcastAddr);
+            configureTransport(gcb, topologyInfo.getMyNodeName(), topologyInfo.getMySiteName(), jgroupsUdpMcastAddr);
             gcb.globalJmxStatistics()
-              .jmxDomain(InfinispanConnectionProvider.JMX_DOMAIN + "-" + nodeName);
+              .jmxDomain(InfinispanConnectionProvider.JMX_DOMAIN + "-" + topologyInfo.getMyNodeName());
-        } else {
-            if (nodeName == null) {
-                nodeName = generateNodeName();
-            }
         }
 
         gcb.globalJmxStatistics()
@@ -208,10 +182,6 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
         cacheManager = new DefaultCacheManager(gcb.build());
         containerManaged = false;
 
-        if (cacheManager.getTransport() != null) {
-            nodeName = cacheManager.getTransport().getAddress().toString();
-        }
-
         logger.debug("Started embedded Infinispan cache container");
 
         ConfigurationBuilder modelCacheConfigBuilder = new ConfigurationBuilder();
@@ -311,7 +281,7 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
         Configuration replicationEvictionCacheConfiguration = replicationConfigBuilder.build();
         cacheManager.defineConfiguration(InfinispanConnectionProvider.WORK_CACHE_NAME, replicationEvictionCacheConfiguration);
 
-        long realmRevisionsMaxEntries = cacheManager.getCache(InfinispanConnectionProvider.REALM_CACHE_NAME).getCacheConfiguration().eviction().maxEntries();
+        long realmRevisionsMaxEntries = cacheManager.getCache(InfinispanConnectionProvider.REALM_CACHE_NAME).getCacheConfiguration().memory().size();
         realmRevisionsMaxEntries = realmRevisionsMaxEntries > 0
                 ? 2 * realmRevisionsMaxEntries
                 : InfinispanConnectionProvider.REALM_REVISIONS_CACHE_DEFAULT_MAX;
@@ -319,7 +289,7 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
         cacheManager.defineConfiguration(InfinispanConnectionProvider.REALM_REVISIONS_CACHE_NAME, getRevisionCacheConfig(realmRevisionsMaxEntries));
         cacheManager.getCache(InfinispanConnectionProvider.REALM_REVISIONS_CACHE_NAME, true);
 
-        long userRevisionsMaxEntries = cacheManager.getCache(InfinispanConnectionProvider.USER_CACHE_NAME).getCacheConfiguration().eviction().maxEntries();
+        long userRevisionsMaxEntries = cacheManager.getCache(InfinispanConnectionProvider.USER_CACHE_NAME).getCacheConfiguration().memory().size();
         userRevisionsMaxEntries = userRevisionsMaxEntries > 0
                 ? 2 * userRevisionsMaxEntries
                 : InfinispanConnectionProvider.USER_REVISIONS_CACHE_DEFAULT_MAX;
@@ -340,7 +310,7 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
         cacheManager.defineConfiguration(InfinispanConnectionProvider.ACTION_TOKEN_CACHE, actionTokenCacheConfigBuilder.build());
         cacheManager.getCache(InfinispanConnectionProvider.ACTION_TOKEN_CACHE, true);
 
-        long authzRevisionsMaxEntries = cacheManager.getCache(InfinispanConnectionProvider.AUTHORIZATION_CACHE_NAME).getCacheConfiguration().eviction().maxEntries();
+        long authzRevisionsMaxEntries = cacheManager.getCache(InfinispanConnectionProvider.AUTHORIZATION_CACHE_NAME).getCacheConfiguration().memory().size();
         authzRevisionsMaxEntries = authzRevisionsMaxEntries > 0
                 ? 2 * authzRevisionsMaxEntries
                 : InfinispanConnectionProvider.AUTHORIZATION_REVISIONS_CACHE_DEFAULT_MAX;
@@ -349,20 +319,21 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
         cacheManager.getCache(InfinispanConnectionProvider.AUTHORIZATION_REVISIONS_CACHE_NAME, true);
     }
 
-    protected String generateNodeName() {
-        return InfinispanConnectionProvider.NODE_PREFIX + new SecureRandom().nextInt(1000000);
-    }
 
     private Configuration getRevisionCacheConfig(long maxEntries) {
         ConfigurationBuilder cb = new ConfigurationBuilder();
         cb.invocationBatching().enable().transaction().transactionMode(TransactionMode.TRANSACTIONAL);
 
-        // Use Dummy manager even in managed ( wildfly/eap ) environment. We don't want infinispan to participate in global transaction
+        // Use Embedded manager even in managed ( wildfly/eap ) environment. We don't want infinispan to participate in global transaction
-        cb.transaction().transactionManagerLookup(new DummyTransactionManagerLookup());
+        cb.transaction().transactionManagerLookup(new EmbeddedTransactionManagerLookup());
 
         cb.transaction().lockingMode(LockingMode.PESSIMISTIC);
 
-        cb.eviction().strategy(EvictionStrategy.LRU).type(EvictionType.COUNT).size(maxEntries);
+        cb.memory()
+                .evictionStrategy(EvictionStrategy.REMOVE)
+                .evictionType(EvictionType.COUNT)
+                .size(maxEntries);
+
         return cb.build();
     }
 
@@ -383,6 +354,7 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
                     .rawValues(true)
                     .forceReturnValues(false)
                     .marshaller(KeycloakHotRodMarshallerFactory.class.getName())
+                    //.protocolVersion(ProtocolVersion.PROTOCOL_VERSION_26)
                     .addServer()
                         .host(jdgServer)
                         .port(jdgPort)
@@ -410,6 +382,7 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
                     .rawValues(true)
                     .forceReturnValues(false)
                     .marshaller(KeycloakHotRodMarshallerFactory.class.getName())
+                    //.protocolVersion(ProtocolVersion.PROTOCOL_VERSION_26)
                     .addServer()
                         .host(jdgServer)
                         .port(jdgPort)
@@ -420,7 +393,12 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
 
     protected Configuration getKeysCacheConfig() {
         ConfigurationBuilder cb = new ConfigurationBuilder();
-        cb.eviction().strategy(EvictionStrategy.LRU).type(EvictionType.COUNT).size(InfinispanConnectionProvider.KEYS_CACHE_DEFAULT_MAX);
+
+        cb.memory()
+                .evictionStrategy(EvictionStrategy.REMOVE)
+                .evictionType(EvictionType.COUNT)
+                .size(InfinispanConnectionProvider.KEYS_CACHE_DEFAULT_MAX);
+
         cb.expiration().maxIdle(InfinispanConnectionProvider.KEYS_CACHE_MAX_IDLE_SECONDS, TimeUnit.SECONDS);
         return cb.build();
     }
@@ -428,9 +406,9 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
     private ConfigurationBuilder getActionTokenCacheConfig() {
         ConfigurationBuilder cb = new ConfigurationBuilder();
 
-        cb.eviction()
+        cb.memory()
-                .strategy(EvictionStrategy.NONE)
+                .evictionStrategy(EvictionStrategy.NONE)
-                .type(EvictionType.COUNT)
+                .evictionType(EvictionType.COUNT)
                 .size(InfinispanConnectionProvider.ACTION_TOKEN_CACHE_DEFAULT_MAX);
         cb.expiration()
                 .maxIdle(InfinispanConnectionProvider.ACTION_TOKEN_MAX_IDLE_SECONDS, TimeUnit.SECONDS)

model/infinispan/src/main/java/org/keycloak/connections/infinispan/InfinispanConnectionProvider.java

@@ -75,14 +75,8 @@ public interface InfinispanConnectionProvider extends Provider {
     <K, V> RemoteCache<K, V> getRemoteCache(String name);
 
     /**
-     * @return Address of current node in cluster. In non-cluster environment, it returns some other non-null value (eg. hostname with some random value like "host-123456" )
+     * @return Information about cluster topology
      */
-    String getNodeName();
+    TopologyInfo getTopologyInfo();
-
-    /**
-     *
-     * @return siteName or null if we're not in environment with multiple sites (data centers)
-     */
-    String getSiteName();
 
 }

model/infinispan/src/main/java/org/keycloak/connections/infinispan/TopologyInfo.java

@@ -0,0 +1,202 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.connections.infinispan;
+
+import java.net.InetSocketAddress;
+import java.security.SecureRandom;
+import java.util.Objects;
+import java.util.Optional;
+
+import org.infinispan.Cache;
+import org.infinispan.distribution.DistributionManager;
+import org.infinispan.manager.EmbeddedCacheManager;
+import org.infinispan.remoting.transport.Address;
+import org.infinispan.remoting.transport.LocalModeAddress;
+import org.infinispan.remoting.transport.Transport;
+import org.infinispan.remoting.transport.jgroups.JGroupsAddress;
+import org.infinispan.remoting.transport.jgroups.JGroupsTransport;
+import org.jboss.logging.Logger;
+import org.jgroups.Event;
+import org.jgroups.JChannel;
+import org.jgroups.stack.IpAddress;
+import org.jgroups.util.NameCache;
+import org.keycloak.Config;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class TopologyInfo {
+
+    private static final Logger logger = Logger.getLogger(TopologyInfo.class);
+
+
+    // Node name used in clustered environment. This typically points to "jboss.node.name" . If "jboss.node.name" is not set, it is randomly generated
+    // name
+    private final String myNodeName;
+
+    // Used just if "site" is configured (typically in cross-dc environment). Otherwise null
+    private final String mySiteName;
+
+    private final boolean isGeneratedNodeName;
+
+
+    public TopologyInfo(EmbeddedCacheManager cacheManager, Config.Scope config, boolean embedded) {
+        String siteName;
+        String nodeName;
+        boolean isGeneratedNodeName = false;
+
+        if (!embedded) {
+            Transport transport = cacheManager.getTransport();
+            if (transport != null) {
+                nodeName = transport.getAddress().toString();
+                siteName = cacheManager.getCacheManagerConfiguration().transport().siteId();
+                if (siteName == null) {
+                    siteName = System.getProperty(InfinispanConnectionProvider.JBOSS_SITE_NAME);
+                }
+            } else {
+                nodeName = System.getProperty(InfinispanConnectionProvider.JBOSS_NODE_NAME);
+                siteName = System.getProperty(InfinispanConnectionProvider.JBOSS_SITE_NAME);
+            }
+            if (nodeName == null || nodeName.equals("localhost")) {
+                isGeneratedNodeName = true;
+                nodeName = generateNodeName();
+            }
+        } else {
+            boolean clustered = config.getBoolean("clustered", false);
+
+            nodeName = config.get("nodeName", System.getProperty(InfinispanConnectionProvider.JBOSS_NODE_NAME));
+            if (nodeName != null && nodeName.isEmpty()) {
+                nodeName = null;
+            }
+
+            siteName = config.get("siteName", System.getProperty(InfinispanConnectionProvider.JBOSS_SITE_NAME));
+            if (siteName != null && siteName.isEmpty()) {
+                siteName = null;
+            }
+
+            if (nodeName == null) {
+                if (!clustered) {
+                    isGeneratedNodeName = true;
+                    nodeName = generateNodeName();
+                } else {
+                    throw new IllegalStateException("You must set jboss.node.name if you use clustered mode for InfinispanConnectionProvider");
+                }
+            }
+        }
+
+        this.myNodeName = nodeName;
+        this.mySiteName = siteName;
+        this.isGeneratedNodeName = isGeneratedNodeName;
+    }
+
+
+    private String generateNodeName() {
+        return InfinispanConnectionProvider.NODE_PREFIX + new SecureRandom().nextInt(1000000);
+    }
+
+
+    public String getMyNodeName() {
+        return myNodeName;
+    }
+
+    public String getMySiteName() {
+        return mySiteName;
+    }
+
+
+    @Override
+    public String toString() {
+        return String.format("Node name: %s, Site name: %s", myNodeName, mySiteName);
+    }
+
+
+    /**
+     * True if I am primary owner of the key in case of distributed caches. In case of local caches, always return true
+     */
+    public boolean amIOwner(Cache cache, Object key) {
+        Address myAddress = cache.getCacheManager().getAddress();
+        Address objectOwnerAddress = getOwnerAddress(cache, key);
+
+        // NOTE: For scattered caches, this will always return true, which may not be correct. Need to review this if we add support for scattered caches
+        return Objects.equals(myAddress, objectOwnerAddress);
+    }
+
+
+    /**
+     * Get route to be used as the identifier for sticky session. Return null if I am not able to find the appropriate route (or in case of local mode)
+     */
+    public String getRouteName(Cache cache, Object key) {
+        if (cache.getCacheConfiguration().clustering().cacheMode().isClustered() && isGeneratedNodeName) {
+            logger.warn("Clustered configuration used, but node name is not properly set. Make sure to start server with jboss.node.name property identifying cluster node");
+        }
+
+        if (isGeneratedNodeName) {
+            return null;
+        }
+
+        // Impl based on Wildfly sticky session algorithm for generating routes ( org.wildfly.clustering.web.infinispan.session.InfinispanRouteLocator )
+        Address address = getOwnerAddress(cache, key);
+
+        // Local mode
+        if (address == null ||  (address == LocalModeAddress.INSTANCE)) {
+            return myNodeName;
+        }
+
+        org.jgroups.Address jgroupsAddress = toJGroupsAddress(address);
+        String name = NameCache.get(jgroupsAddress);
+
+        // If no logical name exists, create one using physical address
+        if (name == null) {
+
+            Transport transport = cache.getCacheManager().getTransport();
+            JChannel jgroupsChannel = ((JGroupsTransport) transport).getChannel();
+
+            IpAddress ipAddress = (IpAddress) jgroupsChannel.down(new Event(Event.GET_PHYSICAL_ADDRESS, jgroupsAddress));
+            // Physical address might be null if node is no longer a member of the cluster
+            InetSocketAddress socketAddress = (ipAddress != null) ? new InetSocketAddress(ipAddress.getIpAddress(), ipAddress.getPort()) : new InetSocketAddress(0);
+            name = String.format("%s:%s", socketAddress.getHostString(), socketAddress.getPort());
+
+            logger.debugf("Address not found in NameCache. Fallback to %s", name);
+        }
+
+        return name;
+    }
+
+
+    private Address getOwnerAddress(Cache cache, Object key) {
+        DistributionManager dist = cache.getAdvancedCache().getDistributionManager();
+        Address address = (dist != null) && !cache.getCacheConfiguration().clustering().cacheMode().isScattered() ?
+                dist.getCacheTopology().getDistribution(key).primary() :
+                cache.getCacheManager().getAddress();
+
+        return address;
+    }
+
+
+    // See org.wildfly.clustering.server.group.CacheGroup
+    private static org.jgroups.Address toJGroupsAddress(Address address) {
+        if ((address == null) || (address == LocalModeAddress.INSTANCE)) return null;
+        if (address instanceof JGroupsAddress) {
+            JGroupsAddress jgroupsAddress = (JGroupsAddress) address;
+            return jgroupsAddress.getJGroupsAddress();
+        }
+        throw new IllegalArgumentException(address.toString());
+    }
+
+
+}

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanStickySessionEncoderProvider.java

@@ -22,6 +22,7 @@ import org.infinispan.distribution.DistributionManager;
 import org.infinispan.remoting.transport.Address;
 import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
 import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.sessions.infinispan.util.InfinispanUtil;
 import org.keycloak.sessions.StickySessionEncoderProvider;
 
 /**
@@ -30,12 +31,10 @@ import org.keycloak.sessions.StickySessionEncoderProvider;
 public class InfinispanStickySessionEncoderProvider implements StickySessionEncoderProvider {
 
     private final KeycloakSession session;
-    private final String myNodeName;
     private final boolean shouldAttachRoute;
 
-    public InfinispanStickySessionEncoderProvider(KeycloakSession session, String myNodeName, boolean shouldAttachRoute) {
+    public InfinispanStickySessionEncoderProvider(KeycloakSession session, boolean shouldAttachRoute) {
         this.session = session;
-        this.myNodeName = myNodeName;
         this.shouldAttachRoute = shouldAttachRoute;
     }
 
@@ -45,9 +44,9 @@ public class InfinispanStickySessionEncoderProvider implements StickySessionEnco
             return sessionId;
         }
 
-        String nodeName = getNodeName(sessionId);
+        String route = getRoute(sessionId);
-        if (nodeName != null) {
+        if (route != null) {
-            return sessionId + '.' + nodeName;
+            return sessionId + '.' + route;
         } else {
             return sessionId;
         }
@@ -71,19 +70,10 @@ public class InfinispanStickySessionEncoderProvider implements StickySessionEnco
     }
 
 
-    private String getNodeName(String sessionId) {
+    private String getRoute(String sessionId) {
         InfinispanConnectionProvider ispnProvider = session.getProvider(InfinispanConnectionProvider.class);
         Cache cache = ispnProvider.getCache(InfinispanConnectionProvider.AUTHENTICATION_SESSIONS_CACHE_NAME);
-        DistributionManager distManager = cache.getAdvancedCache().getDistributionManager();
+        return InfinispanUtil.getTopologyInfo(session).getRouteName(cache, sessionId);
-
-        if (distManager != null) {
-            // Sticky session to the node, who owns this authenticationSession
-            Address address = distManager.getPrimaryLocation(sessionId);
-            return address.toString();
-        } else {
-            // Fallback to jbossNodeName if authSession cache is local
-            return myNodeName;
-        }
     }
 
 

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanStickySessionEncoderProviderFactory.java

@@ -38,15 +38,7 @@ public class InfinispanStickySessionEncoderProviderFactory implements StickySess
 
     @Override
     public StickySessionEncoderProvider create(KeycloakSession session) {
-        String myNodeName = InfinispanUtil.getMyAddress(session);
+        return new InfinispanStickySessionEncoderProvider(session, shouldAttachRoute);
-
-        if (myNodeName != null && myNodeName.startsWith(InfinispanConnectionProvider.NODE_PREFIX)) {
-
-            // Node name was randomly generated. We won't use anything for sticky sessions in this case
-            myNodeName = null;
-        }
-
-        return new InfinispanStickySessionEncoderProvider(session, myNodeName, shouldAttachRoute);
     }
 
     @Override

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProvider.java

@@ -21,7 +21,6 @@ import org.infinispan.Cache;
 import org.infinispan.client.hotrod.RemoteCache;
 import org.infinispan.context.Flag;
 import org.infinispan.stream.CacheCollectors;
-import org.infinispan.stream.SerializableSupplier;
 import org.jboss.logging.Logger;
 import org.keycloak.cluster.ClusterProvider;
 import org.keycloak.common.util.Time;

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProviderFactory.java

@@ -141,7 +141,7 @@ public class InfinispanUserSessionProviderFactory implements UserSessionProvider
 
     // Count of sessions to be computed in each segment
     private int getSessionsPerSegment() {
-        return config.getInt("sessionsPerSegment", 100);
+        return config.getInt("sessionsPerSegment", 64);
     }
 
     private int getTimeoutForPreloadingSessionsSeconds() {
@@ -161,7 +161,8 @@ public class InfinispanUserSessionProviderFactory implements UserSessionProvider
                 InfinispanConnectionProvider connections = session.getProvider(InfinispanConnectionProvider.class);
                 Cache<String, Serializable> workCache = connections.getCache(InfinispanConnectionProvider.WORK_CACHE_NAME);
 
-                InfinispanCacheInitializer ispnInitializer = new InfinispanCacheInitializer(sessionFactory, workCache, new OfflinePersistentUserSessionLoader(), "offlineUserSessions", sessionsPerSegment, maxErrors);
+                InfinispanCacheInitializer ispnInitializer = new InfinispanCacheInitializer(sessionFactory, workCache,
+                        new OfflinePersistentUserSessionLoader(sessionsPerSegment), "offlineUserSessions", sessionsPerSegment, maxErrors);
 
                 // DB-lock to ensure that persistent sessions are loaded from DB just on one DC. The other DCs will load them from remote cache.
                 CacheInitializer initializer = new DBLockBasedCacheInitializer(session, ispnInitializer);
@@ -302,7 +303,8 @@ public class InfinispanUserSessionProviderFactory implements UserSessionProvider
                 InfinispanConnectionProvider connections = session.getProvider(InfinispanConnectionProvider.class);
                 Cache<String, Serializable> workCache = connections.getCache(InfinispanConnectionProvider.WORK_CACHE_NAME);
 
-                InfinispanCacheInitializer initializer = new InfinispanCacheInitializer(sessionFactory, workCache, new RemoteCacheSessionsLoader(cacheName), "remoteCacheLoad::" + cacheName, sessionsPerSegment, maxErrors);
+                InfinispanCacheInitializer initializer = new InfinispanCacheInitializer(sessionFactory, workCache,
+                        new RemoteCacheSessionsLoader(cacheName, sessionsPerSegment), "remoteCacheLoad::" + cacheName, sessionsPerSegment, maxErrors);
 
                 initializer.initCache();
                 initializer.loadSessions();

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/changes/SessionEntityWrapper.java

@@ -185,15 +185,17 @@ public class SessionEntityWrapper<S extends SessionEntity> {
             if (forTransport) {
                 final SessionEntity entity = (SessionEntity) input.readObject();
                 final SessionEntityWrapper res = new SessionEntityWrapper(entity);
-                if (log.isDebugEnabled()) {
+                if (log.isTraceEnabled()) {
-                    log.debugf("Loaded entity from remote store: %s, version=%s, metadata=%s", entity, res.version, res.localMetadata);
+                    log.tracef("Loaded entity from remote store: %s, version=%s, metadata=%s", entity, res.version, res.localMetadata);
                 }
                 return res;
             } else {
                 UUID sessionVersion = new UUID(input.readLong(), input.readLong());
                 HashMap<String, String> map = MarshallUtil.unmarshallMap(input, HashMap::new);
                 final SessionEntity entity = (SessionEntity) input.readObject();
-                log.debugf("Found entity locally: entity=%s, version=%s, metadata=%s", entity, sessionVersion, map);
+                if (log.isTraceEnabled()) {
+                    log.tracef("Found entity locally: entity=%s, version=%s, metadata=%s", entity, sessionVersion, map);
+                }
                 return new SessionEntityWrapper(sessionVersion, map, entity);
             }
         }

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/changes/sessions/LastSessionRefreshListener.java

@@ -23,6 +23,7 @@ import org.infinispan.Cache;
 import org.jboss.logging.Logger;
 import org.keycloak.cluster.ClusterEvent;
 import org.keycloak.cluster.ClusterListener;
+import org.keycloak.connections.infinispan.TopologyInfo;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
 import org.keycloak.models.RealmModel;
@@ -45,20 +46,14 @@ public class LastSessionRefreshListener implements ClusterListener {
 
     private final KeycloakSessionFactory sessionFactory;
     private final Cache<String, SessionEntityWrapper<UserSessionEntity>> cache;
-    private final boolean distributed;
+    private final TopologyInfo topologyInfo;
-    private final String myAddress;
 
     public LastSessionRefreshListener(KeycloakSession session, Cache<String, SessionEntityWrapper<UserSessionEntity>> cache, boolean offline) {
         this.sessionFactory = session.getKeycloakSessionFactory();
         this.cache = cache;
         this.offline = offline;
 
-        this.distributed = InfinispanUtil.isDistributedCache(cache);
+        this.topologyInfo = InfinispanUtil.getTopologyInfo(session);
-        if (this.distributed) {
-            this.myAddress = InfinispanUtil.getMyAddress(session);
-        } else {
-            this.myAddress = null;
-        }
     }
 
     @Override
@@ -81,7 +76,7 @@ public class LastSessionRefreshListener implements ClusterListener {
                     RealmModel realm = kcSession.realms().getRealm(realmId);
                     UserSessionModel userSession = offline ? kcSession.sessions().getOfflineUserSession(realm, sessionId) : kcSession.sessions().getUserSession(realm, sessionId);
                     if (userSession == null) {
-                        logger.debugf("User session '%s' not available on node '%s' offline '%b'", sessionId, myAddress, offline);
+                        logger.debugf("User session '%s' not available on node '%s' offline '%b'", sessionId, topologyInfo.getMyNodeName(), offline);
                     } else {
                         // Update just if lastSessionRefresh from event is bigger than ours
                         if (lastSessionRefresh > userSession.getLastSessionRefresh()) {
@@ -101,11 +96,6 @@ public class LastSessionRefreshListener implements ClusterListener {
 
     // For distributed caches, ensure that local modification is executed just on owner
     protected boolean shouldUpdateLocalCache(String key) {
-        if (!distributed) {
+        return topologyInfo.amIOwner(cache, key);
-            return true;
-        } else {
-            String keyAddress = InfinispanUtil.getKeyPrimaryOwnerAddress(cache, key);
-            return myAddress.equals(keyAddress);
-        }
     }
 }

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/changes/sessions/SessionData.java

@@ -24,6 +24,7 @@ import java.io.ObjectOutput;
 import org.infinispan.commons.marshall.Externalizer;
 import org.infinispan.commons.marshall.MarshallUtil;
 import org.infinispan.commons.marshall.SerializeWith;
+import org.keycloak.models.sessions.infinispan.util.KeycloakMarshallUtil;
 
 /**
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
@@ -58,14 +59,14 @@ public class SessionData {
         @Override
         public void writeObject(ObjectOutput output, SessionData obj) throws IOException {
             MarshallUtil.marshallString(obj.realmId, output);
-            MarshallUtil.marshallInt(output, obj.lastSessionRefresh);
+            KeycloakMarshallUtil.marshall(obj.lastSessionRefresh, output);
         }
 
 
         @Override
         public SessionData readObject(ObjectInput input) throws IOException, ClassNotFoundException {
             String realmId = MarshallUtil.unmarshallString(input);
-            int lastSessionRefresh = MarshallUtil.unmarshallInt(input);
+            int lastSessionRefresh = KeycloakMarshallUtil.unmarshallInteger(input);
 
             return new SessionData(realmId, lastSessionRefresh);
         }

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/entities/AuthenticatedClientSessionEntity.java

@@ -174,14 +174,14 @@ public class AuthenticatedClientSessionEntity extends SessionEntity {
             MarshallUtil.marshallString(session.getRealmId(), output);
             MarshallUtil.marshallString(session.getAuthMethod(), output);
             MarshallUtil.marshallString(session.getRedirectUri(), output);
-            MarshallUtil.marshallInt(output, session.getTimestamp());
+            KeycloakMarshallUtil.marshall(session.getTimestamp(), output);
             MarshallUtil.marshallString(session.getAction(), output);
 
             Map<String, String> notes = session.getNotes();
             KeycloakMarshallUtil.writeMap(notes, KeycloakMarshallUtil.STRING_EXT, KeycloakMarshallUtil.STRING_EXT, output);
 
             MarshallUtil.marshallString(session.getCurrentRefreshToken(), output);
-            MarshallUtil.marshallInt(output, session.getCurrentRefreshTokenUseCount());
+            KeycloakMarshallUtil.marshall(session.getCurrentRefreshTokenUseCount(), output);
         }
 
 
@@ -193,7 +193,7 @@ public class AuthenticatedClientSessionEntity extends SessionEntity {
 
             sessionEntity.setAuthMethod(MarshallUtil.unmarshallString(input));
             sessionEntity.setRedirectUri(MarshallUtil.unmarshallString(input));
-            sessionEntity.setTimestamp(MarshallUtil.unmarshallInt(input));
+            sessionEntity.setTimestamp(KeycloakMarshallUtil.unmarshallInteger(input));
             sessionEntity.setAction(MarshallUtil.unmarshallString(input));
 
             Map<String, String> notes = KeycloakMarshallUtil.readMap(input, KeycloakMarshallUtil.STRING_EXT, KeycloakMarshallUtil.STRING_EXT,
@@ -201,7 +201,7 @@ public class AuthenticatedClientSessionEntity extends SessionEntity {
             sessionEntity.setNotes(notes);
 
             sessionEntity.setCurrentRefreshToken(MarshallUtil.unmarshallString(input));
-            sessionEntity.setCurrentRefreshTokenUseCount(MarshallUtil.unmarshallInt(input));
+            sessionEntity.setCurrentRefreshTokenUseCount(KeycloakMarshallUtil.unmarshallInteger(input));
 
             return sessionEntity;
         }

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/entities/AuthenticationSessionEntity.java

@@ -23,7 +23,7 @@ import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 
-import org.infinispan.util.concurrent.ConcurrentHashSet;
+import org.infinispan.commons.util.concurrent.ConcurrentHashSet;
 import org.keycloak.sessions.AuthenticationSessionModel;
 import org.keycloak.sessions.CommonClientSessionModel.ExecutionStatus;
 import java.io.IOException;

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/entities/UserSessionEntity.java

@@ -254,8 +254,8 @@ public class UserSessionEntity extends SessionEntity {
             MarshallUtil.marshallString(session.getRealmId(), output);
             MarshallUtil.marshallString(session.getUser(), output);
 
-            MarshallUtil.marshallInt(output, session.getLastSessionRefresh());
+            KeycloakMarshallUtil.marshall(session.getLastSessionRefresh(), output);
-            MarshallUtil.marshallInt(output, session.getStarted());
+            KeycloakMarshallUtil.marshall(session.getStarted(), output);
             output.writeBoolean(session.isRememberMe());
 
             int state = session.getState() == null ? 0 : STATE_TO_ID.get(session.getState());
@@ -291,8 +291,8 @@ public class UserSessionEntity extends SessionEntity {
             sessionEntity.setRealmId(MarshallUtil.unmarshallString(input));
             sessionEntity.setUser(MarshallUtil.unmarshallString(input));
 
-            sessionEntity.setLastSessionRefresh(MarshallUtil.unmarshallInt(input));
+            sessionEntity.setLastSessionRefresh(KeycloakMarshallUtil.unmarshallInteger(input));
-            sessionEntity.setStarted(MarshallUtil.unmarshallInt(input));
+            sessionEntity.setStarted(KeycloakMarshallUtil.unmarshallInteger(input));
             sessionEntity.setRememberMe(input.readBoolean());
 
             sessionEntity.setState(ID_TO_STATE.get(input.readInt()));

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/events/AbstractUserSessionClusterListener.java

@@ -21,6 +21,7 @@ import org.jboss.logging.Logger;
 import org.keycloak.cluster.ClusterEvent;
 import org.keycloak.cluster.ClusterListener;
 import org.keycloak.cluster.ClusterProvider;
+import org.keycloak.connections.infinispan.TopologyInfo;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
 import org.keycloak.models.UserSessionProvider;
@@ -73,8 +74,9 @@ public abstract class AbstractUserSessionClusterListener<SE extends SessionClust
         }
 
         // Just the initiator will re-send the event after receiving it
-        String myNode = InfinispanUtil.getMyAddress(session);
+        TopologyInfo topology = InfinispanUtil.getTopologyInfo(session);
-        String mySite = InfinispanUtil.getMySite(session);
+        String myNode = topology.getMyNodeName();
+        String mySite = topology.getMySiteName();
         return (event.getNodeId() != null && event.getNodeId().equals(myNode) && event.getSiteId() != null && event.getSiteId().equals(mySite));
     }
 

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/events/SessionClusterEvent.java

@@ -18,6 +18,7 @@
 package org.keycloak.models.sessions.infinispan.events;
 
 import org.keycloak.cluster.ClusterEvent;
+import org.keycloak.connections.infinispan.TopologyInfo;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.sessions.infinispan.util.InfinispanUtil;
 import java.io.IOException;
@@ -52,8 +53,9 @@ public abstract class SessionClusterEvent implements ClusterEvent {
         this.realmId = realmId;
         this.eventKey = eventKey;
         this.resendingEvent = resendingEvent;
-        this.siteId = InfinispanUtil.getMySite(session);
+        TopologyInfo topology = InfinispanUtil.getTopologyInfo(session);
-        this.nodeId = InfinispanUtil.getMyAddress(session);
+        this.siteId = topology.getMySiteName();
+        this.nodeId = topology.getMyNodeName();
     }
 
 

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/initializer/BaseCacheInitializer.java

@@ -73,38 +73,7 @@ public abstract class BaseCacheInitializer extends CacheInitializer {
     }
 
 
-    protected InitializerState getOrCreateInitializerState() {
+    protected InitializerState getStateFromCache() {
-        InitializerState state = getStateFromCache();
-        if (state == null) {
-            final int[] count = new int[1];
-
-            // Rather use separate transactions for update and counting
-
-            KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
-                @Override
-                public void run(KeycloakSession session) {
-                    sessionLoader.init(session);
-                }
-
-            });
-
-            KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
-                @Override
-                public void run(KeycloakSession session) {
-                    count[0] = sessionLoader.getSessionsCount(session);
-                }
-
-            });
-
-            state = new InitializerState(count[0], sessionsPerSegment);
-            saveStateToCache(state);
-        }
-        return state;
-
-    }
-
-
-    private InitializerState getStateFromCache() {
         // We ignore cacheStore for now, so that in Cross-DC scenario (with RemoteStore enabled) is the remoteStore ignored.
         return (InitializerState) workCache.getAdvancedCache()
                 .withFlags(Flag.SKIP_CACHE_STORE, Flag.SKIP_CACHE_LOAD)

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/initializer/InfinispanCacheInitializer.java

@@ -21,7 +21,10 @@ import org.infinispan.Cache;
 import org.infinispan.distexec.DefaultExecutorService;
 import org.infinispan.remoting.transport.Transport;
 import org.jboss.logging.Logger;
+import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
+import org.keycloak.models.KeycloakSessionTask;
+import org.keycloak.models.utils.KeycloakModelUtils;
 
 import java.io.Serializable;
 import java.util.LinkedList;
@@ -60,8 +63,46 @@ public class InfinispanCacheInitializer extends BaseCacheInitializer {
     // Just coordinator will run this
     @Override
     protected void startLoading() {
-        InitializerState state = getOrCreateInitializerState();
+        InitializerState state = getStateFromCache();
+        SessionLoader.LoaderContext[] ctx = new SessionLoader.LoaderContext[1];
+        if (state == null) {
+            // Rather use separate transactions for update and counting
+            KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
+                @Override
+                public void run(KeycloakSession session) {
+                    sessionLoader.init(session);
+                }
 
+            });
+
+            KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
+                @Override
+                public void run(KeycloakSession session) {
+                    ctx[0] = sessionLoader.computeLoaderContext(session);
+                }
+
+            });
+
+            state = new InitializerState(ctx[0].getSegmentsCount());
+            saveStateToCache(state);
+        } else {
+            KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
+                @Override
+                public void run(KeycloakSession session) {
+                    ctx[0] = sessionLoader.computeLoaderContext(session);
+                }
+
+            });
+        }
+
+        log.debugf("Start loading with loader: '%s', ctx: '%s' , state: %s",
+                sessionLoader.toString(), ctx[0].toString(), state.toString());
+
+        startLoadingImpl(state, ctx[0]);
+    }
+
+
+    protected void startLoadingImpl(InitializerState state, SessionLoader.LoaderContext ctx) {
         // Assume each worker has same processor's count
         int processors = Runtime.getRuntime().availableProcessors();
 
@@ -88,7 +129,7 @@ public class InfinispanCacheInitializer extends BaseCacheInitializer {
                 List<Future<WorkerResult>> futures = new LinkedList<>();
                 for (Integer segment : segments) {
                     SessionInitializerWorker worker = new SessionInitializerWorker();
-                    worker.setWorkerEnvironment(segment, sessionsPerSegment, sessionLoader);
+                    worker.setWorkerEnvironment(segment, ctx, sessionLoader);
                     if (!distributed) {
                         worker.setEnvironment(workCache, null);
                     }

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/initializer/InitializerState.java

@@ -42,33 +42,25 @@ public class InitializerState extends SessionEntity {
 
     private static final Logger log = Logger.getLogger(InitializerState.class);
 
-    private final int sessionsCount;
     private final int segmentsCount;
     private final BitSet segments;
     private int lowestUnfinishedSegment = 0;
 
-    public InitializerState(int sessionsCount, int sessionsPerSegment) {
+    public InitializerState(int segmentsCount) {
-        this.sessionsCount = sessionsCount;
+        this.segmentsCount = segmentsCount;
+        this.segments = new BitSet(segmentsCount);
 
-        int segmentsCountLocal = sessionsCount / sessionsPerSegment;
+        log.debugf("segmentsCount: %d", segmentsCount);
-        if (sessionsPerSegment * segmentsCountLocal < sessionsCount) {
-            segmentsCountLocal = segmentsCountLocal + 1;
-        }
-        this.segmentsCount = segmentsCountLocal;
-        this.segments = new BitSet(segmentsCountLocal);
-
-        log.debugf("sessionsCount: %d, sessionsPerSegment: %d, segmentsCount: %d", sessionsCount, sessionsPerSegment, segmentsCountLocal);
 
         updateLowestUnfinishedSegment();
     }
 
-    private InitializerState(String realmId, int sessionsCount, int segmentsCount, BitSet segments) {
+    private InitializerState(String realmId, int segmentsCount, BitSet segments) {
         super(realmId);
-        this.sessionsCount = sessionsCount;
         this.segmentsCount = segmentsCount;
         this.segments = segments;
 
-        log.debugf("sessionsCount: %d, segmentsCount: %d", sessionsCount, segmentsCount);
+        log.debugf("segmentsCount: %d", segmentsCount);
 
         updateLowestUnfinishedSegment();
     }
@@ -116,18 +108,15 @@ public class InitializerState extends SessionEntity {
     @Override
     public String toString() {
         int finished = segments.cardinality();
-        int nonFinished = this.segmentsCount;
+        int nonFinished = segmentsCount - finished;
 
-        return "sessionsCount: "
+        return "finished segments count: " + finished
-          + sessionsCount
-          + (", finished segments count: " + finished)
           + (", non-finished segments count: " + nonFinished);
     }
 
     @Override
     public int hashCode() {
         int hash = 3;
-        hash = 97 * hash + this.sessionsCount;
         hash = 97 * hash + this.segmentsCount;
         hash = 97 * hash + Objects.hashCode(this.segments);
         hash = 97 * hash + this.lowestUnfinishedSegment;
@@ -146,9 +135,6 @@ public class InitializerState extends SessionEntity {
             return false;
         }
         final InitializerState other = (InitializerState) obj;
-        if (this.sessionsCount != other.sessionsCount) {
-            return false;
-        }
         if (this.segmentsCount != other.segmentsCount) {
             return false;
         }
@@ -170,7 +156,6 @@ public class InitializerState extends SessionEntity {
             output.writeByte(VERSION_1);
 
             MarshallUtil.marshallString(value.getRealmId(), output);
-            output.writeInt(value.sessionsCount);
             output.writeInt(value.segmentsCount);
             MarshallUtil.marshallByteArray(value.segments.toByteArray(), output);
         }
@@ -189,7 +174,6 @@ public class InitializerState extends SessionEntity {
             return new InitializerState(
               MarshallUtil.unmarshallString(input),
               input.readInt(),
-              input.readInt(),
               BitSet.valueOf(MarshallUtil.unmarshallByteArray(input))
             );
         }

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/initializer/OfflinePersistentUserSessionLoader.java

@@ -29,12 +29,11 @@ import org.keycloak.models.session.UserSessionPersisterProvider;
 
 import java.io.Serializable;
 import java.util.List;
-import java.util.concurrent.TimeUnit;
 
 /**
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
-public class OfflinePersistentUserSessionLoader implements SessionLoader, Serializable {
+public class OfflinePersistentUserSessionLoader implements SessionLoader<OfflinePersistentUserSessionLoaderContext>, Serializable {
 
     private static final Logger log = Logger.getLogger(OfflinePersistentUserSessionLoader.class);
 
@@ -45,6 +44,13 @@ public class OfflinePersistentUserSessionLoader implements SessionLoader, Serial
     public static final String PERSISTENT_SESSIONS_LOADED_IN_CURRENT_DC = "PERSISTENT_SESSIONS_LOADED_IN_CURRENT_DC";
 
 
+    private final int sessionsPerSegment;
+
+    public OfflinePersistentUserSessionLoader(int sessionsPerSegment) {
+        this.sessionsPerSegment = sessionsPerSegment;
+    }
+
+
     @Override
     public void init(KeycloakSession session) {
         UserSessionPersisterProvider persister = session.getProvider(UserSessionPersisterProvider.class);
@@ -60,14 +66,19 @@ public class OfflinePersistentUserSessionLoader implements SessionLoader, Serial
 
 
     @Override
-    public int getSessionsCount(KeycloakSession session) {
+    public OfflinePersistentUserSessionLoaderContext computeLoaderContext(KeycloakSession session) {
         UserSessionPersisterProvider persister = session.getProvider(UserSessionPersisterProvider.class);
-        return persister.getUserSessionsCount(true);
+        int sessionsCount = persister.getUserSessionsCount(true);
+
+        return new OfflinePersistentUserSessionLoaderContext(sessionsCount, sessionsPerSegment);
     }
 
 
     @Override
-    public boolean loadSessions(KeycloakSession session, int first, int max) {
+    public boolean loadSessions(KeycloakSession session, OfflinePersistentUserSessionLoaderContext ctx, int segment) {
+        int first = ctx.getSessionsPerSegment() * segment;
+        int max = sessionsPerSegment;
+
         if (log.isTraceEnabled()) {
             log.tracef("Loading sessions - first: %d, max: %d", first, max);
         }
@@ -132,4 +143,13 @@ public class OfflinePersistentUserSessionLoader implements SessionLoader, Serial
         log.debugf("Persistent sessions loaded successfully!");
     }
 
+
+    @Override
+    public String toString() {
+        return new StringBuilder("OfflinePersistentUserSessionLoader [ ")
+                .append("sessionsPerSegment: ").append(sessionsPerSegment)
+                .append(" ]")
+                .toString();
+    }
+
 }

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/initializer/OfflinePersistentUserSessionLoaderContext.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.models.sessions.infinispan.initializer;
+
+import java.io.Serializable;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class OfflinePersistentUserSessionLoaderContext implements SessionLoader.LoaderContext, Serializable {
+
+    private final int sessionsTotal;
+    private final int segmentsCount;
+    private final int sessionsPerSegment;
+
+    public OfflinePersistentUserSessionLoaderContext(int sessionsTotal, int sessionsPerSegment) {
+        this.sessionsTotal = sessionsTotal;
+        this.sessionsPerSegment = sessionsPerSegment;
+
+        int segmentsCount = sessionsTotal / sessionsPerSegment;
+        if (sessionsTotal % sessionsPerSegment >= 1) {
+            segmentsCount = segmentsCount + 1;
+        }
+        this.segmentsCount = segmentsCount;
+    }
+
+
+    public int getSessionsTotal() {
+        return sessionsTotal;
+    }
+
+    @Override
+    public int getSegmentsCount() {
+        return segmentsCount;
+    }
+
+    public int getSessionsPerSegment() {
+        return sessionsPerSegment;
+    }
+
+
+    @Override
+    public String toString() {
+        return new StringBuilder("OfflinePersistentUserSessionLoaderContext [ ")
+                .append(" sessionsTotal: ").append(sessionsTotal)
+                .append(", sessionsPerSegment: ").append(sessionsPerSegment)
+                .append(", segmentsCount: ").append(segmentsCount)
+                .append(" ]")
+                .toString();
+    }
+}

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/initializer/SessionInitializerWorker.java

@@ -36,14 +36,14 @@ public class SessionInitializerWorker implements DistributedCallable<String, Ser
     private static final Logger log = Logger.getLogger(SessionInitializerWorker.class);
 
     private int segment;
-    private int sessionsPerSegment;
+    private SessionLoader.LoaderContext ctx;
     private SessionLoader sessionLoader;
 
     private transient Cache<String, Serializable> workCache;
 
-    public void setWorkerEnvironment(int segment, int sessionsPerSegment, SessionLoader sessionLoader) {
+    public void setWorkerEnvironment(int segment, SessionLoader.LoaderContext ctx, SessionLoader sessionLoader) {
         this.segment = segment;
-        this.sessionsPerSegment = sessionsPerSegment;
+        this.ctx = ctx;
         this.sessionLoader = sessionLoader;
     }
 
@@ -64,14 +64,11 @@ public class SessionInitializerWorker implements DistributedCallable<String, Ser
             return InfinispanCacheInitializer.WorkerResult.create(segment, false);
         }
 
-        final int first = segment * sessionsPerSegment;
-        final int max = sessionsPerSegment;
-
         KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
 
             @Override
             public void run(KeycloakSession session) {
-                sessionLoader.loadSessions(session, first, max);
+                sessionLoader.loadSessions(session, ctx, segment);
             }
 
         });

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/initializer/SessionLoader.java

@@ -17,12 +17,14 @@
 
 package org.keycloak.models.sessions.infinispan.initializer;
 
+import java.io.Serializable;
+
 import org.keycloak.models.KeycloakSession;
 
 /**
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
-public interface SessionLoader {
+public interface SessionLoader<LOADER_CONTEXT extends SessionLoader.LoaderContext> extends Serializable {
 
     /**
      * Will be triggered just once on cluster coordinator node to perform some generic initialization tasks (Eg. update DB before starting load).
@@ -35,23 +37,27 @@ public interface SessionLoader {
 
 
     /**
-     * Will be triggered just once on cluster coordinator node to count the number of sessions
+     *
+     * Will be triggered just once on cluster coordinator node to count the number of segments and other context data specific to the worker task.
+     * Each segment will be then later computed in one "worker" task
+     *
+     * This method could be expensive to call, so the "computed" loaderContext object is passed among workers/loaders and needs to be serializable
      *
      * @param session
      * @return
      */
-    int getSessionsCount(KeycloakSession session);
+    LOADER_CONTEXT computeLoaderContext(KeycloakSession session);
 
 
     /**
      * Will be called on all cluster nodes to load the specified page.
      *
      * @param session
-     * @param first
+     * @param loaderContext loaderContext object, which was already computed before
-     * @param max
+     * @param segment to be computed
      * @return
      */
-    boolean loadSessions(KeycloakSession session, int first, int max);
+    boolean loadSessions(KeycloakSession session, LOADER_CONTEXT loaderContext, int segment);
 
 
     /**
@@ -69,4 +75,15 @@ public interface SessionLoader {
      * @param initializer
      */
     void afterAllSessionsLoaded(BaseCacheInitializer initializer);
+
+
+    /**
+     * Object, which contains some context data to be used by SessionLoader implementation. It's computed just once and then passed
+     * to each {@link SessionLoader}. It needs to be {@link Serializable}
+     */
+    interface LoaderContext extends Serializable {
+
+        int getSegmentsCount();
+
+    }
 }

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/initializer/SingleWorkerCacheInitializer.java

@@ -1,51 +0,0 @@
-/*
- * Copyright 2016 Red Hat, Inc. and/or its affiliates
- * and other contributors as indicated by the @author tags.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.keycloak.models.sessions.infinispan.initializer;
-
-import java.io.Serializable;
-
-import org.infinispan.Cache;
-import org.keycloak.models.KeycloakSession;
-
-/**
- * This impl is able to run the non-paginatable loader task and hence will be executed just on single node.
- *
- * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
- */
-public class SingleWorkerCacheInitializer extends BaseCacheInitializer {
-
-    private final KeycloakSession session;
-
-    public SingleWorkerCacheInitializer(KeycloakSession session, Cache<String, Serializable> workCache, SessionLoader sessionLoader, String stateKeySuffix) {
-        super(session.getKeycloakSessionFactory(), workCache, sessionLoader, stateKeySuffix, Integer.MAX_VALUE);
-        this.session = session;
-    }
-
-    @Override
-    protected void startLoading() {
-        InitializerState state = getOrCreateInitializerState();
-        while (!state.isFinished()) {
-            sessionLoader.loadSessions(session, -1, -1);
-            state.markSegmentFinished(0);
-            saveStateToCache(state);
-        }
-
-        // Loader callback after the task is finished
-        this.sessionLoader.afterAllSessionsLoaded(this);
-    }
-}

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/RemoteCacheInvoker.java

@@ -29,6 +29,7 @@ import org.infinispan.client.hotrod.Flag;
 import org.infinispan.client.hotrod.RemoteCache;
 import org.infinispan.client.hotrod.VersionedValue;
 import org.jboss.logging.Logger;
+import org.keycloak.connections.infinispan.TopologyInfo;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.RealmModel;
 import org.keycloak.models.sessions.infinispan.changes.SessionEntityWrapper;
@@ -83,10 +84,12 @@ public class RemoteCacheInvoker {
             logger.tracef("Running task '%s' on remote cache '%s' . Key is '%s'", operation, cacheName, key);
         }
 
+        TopologyInfo topology = InfinispanUtil.getTopologyInfo(kcSession);
+
         Retry.executeWithBackoff((int iteration) -> {
 
             try {
-                runOnRemoteCache(context.remoteCache, maxIdleTimeMs, key, task, sessionWrapper);
+                runOnRemoteCache(topology, context.remoteCache, maxIdleTimeMs, key, task, sessionWrapper);
             } catch (HotRodClientException re) {
                 if (logger.isDebugEnabled()) {
                     logger.debugf(re, "Failed running task '%s' on remote cache '%s' . Key: '%s', iteration '%s'. Will try to retry the task",
@@ -101,7 +104,7 @@ public class RemoteCacheInvoker {
     }
 
 
-    private <K, V extends SessionEntity> void runOnRemoteCache(RemoteCache<K, SessionEntityWrapper<V>> remoteCache, long maxIdleMs, K key, SessionUpdateTask<V> task, SessionEntityWrapper<V> sessionWrapper) {
+    private <K, V extends SessionEntity> void runOnRemoteCache(TopologyInfo topology, RemoteCache<K, SessionEntityWrapper<V>> remoteCache, long maxIdleMs, K key, SessionUpdateTask<V> task, SessionEntityWrapper<V> sessionWrapper) {
         final V session = sessionWrapper.getEntity();
         SessionUpdateTask.CacheOperation operation = task.getOperation(session);
 
@@ -119,11 +122,11 @@ public class RemoteCacheInvoker {
                 if (existing != null) {
                     logger.debugf("Existing entity in remote cache for key: %s . Will update it", key);
 
-                    replace(remoteCache, task.getLifespanMs(), maxIdleMs, key, task);
+                    replace(topology, remoteCache, task.getLifespanMs(), maxIdleMs, key, task);
                 }
                 break;
             case REPLACE:
-                replace(remoteCache, task.getLifespanMs(), maxIdleMs, key, task);
+                replace(topology, remoteCache, task.getLifespanMs(), maxIdleMs, key, task);
                 break;
             default:
                 throw new IllegalStateException("Unsupported state " +  operation);
@@ -131,14 +134,13 @@ public class RemoteCacheInvoker {
     }
 
 
-    private <K, V extends SessionEntity> void replace(RemoteCache<K, SessionEntityWrapper<V>> remoteCache, long lifespanMs, long maxIdleMs, K key, SessionUpdateTask<V> task) {
+    private <K, V extends SessionEntity> void replace(TopologyInfo topology, RemoteCache<K, SessionEntityWrapper<V>> remoteCache, long lifespanMs, long maxIdleMs, K key, SessionUpdateTask<V> task) {
         boolean replaced = false;
-        int iteration = 0;
+        int replaceIteration = 0;
+        while (!replaced && replaceIteration < InfinispanUtil.MAXIMUM_REPLACE_RETRIES) {
+            replaceIteration++;
 
-        while (!replaced && iteration < InfinispanUtil.MAXIMUM_REPLACE_RETRIES) {
+            VersionedValue<SessionEntityWrapper<V>> versioned = remoteCache.getWithMetadata(key);
-            iteration++;
-
-            VersionedValue<SessionEntityWrapper<V>> versioned = remoteCache.getVersioned(key);
             if (versioned == null) {
                 logger.warnf("Not found entity to replace for key '%s'", key);
                 return;
@@ -151,16 +153,17 @@ public class RemoteCacheInvoker {
             task.runUpdate(session);
 
             if (logger.isTraceEnabled()) {
-                logger.tracef("Before replaceWithVersion. Entity to write version %d: %s", versioned.getVersion(), session);
+                logger.tracef("%s: Before replaceWithVersion. Entity to write version %d: %s", logTopologyData(topology, replaceIteration),
+                        versioned.getVersion(), session);
             }
 
             replaced = remoteCache.replaceWithVersion(key, SessionEntityWrapper.forTransport(session), versioned.getVersion(), lifespanMs, TimeUnit.MILLISECONDS, maxIdleMs, TimeUnit.MILLISECONDS);
 
             if (!replaced) {
-                logger.debugf("Failed to replace entity '%s' version %d. Will retry again", key, versioned.getVersion());
+                logger.debugf("%s: Failed to replace entity '%s' version %d. Will retry again", logTopologyData(topology, replaceIteration), key, versioned.getVersion());
             } else {
                 if (logger.isTraceEnabled()) {
-                    logger.tracef("Replaced entity version %d in remote cache: %s", versioned.getVersion(), session);
+                    logger.tracef("%s: Replaced entity version %d in remote cache: %s", logTopologyData(topology, replaceIteration), versioned.getVersion(), session);
                 }
             }
         }
@@ -171,6 +174,11 @@ public class RemoteCacheInvoker {
     }
 
 
+    private String logTopologyData(TopologyInfo topology, int iteration) {
+        return topology.toString() + ", replaceIteration: " + iteration;
+    }
+
+
     private class RemoteCacheContext {
 
         private final RemoteCache remoteCache;

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/RemoteCacheSessionListener.java

@@ -29,6 +29,7 @@ import org.infinispan.client.hotrod.event.ClientCacheEntryRemovedEvent;
 import org.infinispan.client.hotrod.event.ClientEvent;
 import org.infinispan.context.Flag;
 import org.jboss.logging.Logger;
+import org.keycloak.connections.infinispan.TopologyInfo;
 import org.keycloak.executors.ExecutorsProvider;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.sessions.infinispan.changes.SessionEntityWrapper;
@@ -46,10 +47,11 @@ public class RemoteCacheSessionListener<K, V extends SessionEntity>  {
 
     protected static final Logger logger = Logger.getLogger(RemoteCacheSessionListener.class);
 
+    private static final int MAXIMUM_REPLACE_RETRIES = 10;
+
     private Cache<K, SessionEntityWrapper<V>> cache;
     private RemoteCache<K, SessionEntityWrapper<V>> remoteCache;
-    private boolean distributed;
+    private TopologyInfo topologyInfo;
-    private String myAddress;
     private ClientListenerExecutorDecorator<K> executor;
 
 
@@ -61,12 +63,7 @@ public class RemoteCacheSessionListener<K, V extends SessionEntity>  {
         this.cache = cache;
         this.remoteCache = remoteCache;
 
-        this.distributed = InfinispanUtil.isDistributedCache(cache);
+        this.topologyInfo = InfinispanUtil.getTopologyInfo(session);
-        if (this.distributed) {
-            this.myAddress = InfinispanUtil.getMyAddress(session);
-        } else {
-            this.myAddress = null;
-        }
 
         ExecutorService executor = session.getProvider(ExecutorsProvider.class).getExecutor("client-listener-" + cache.getName());
         this.executor = new ClientListenerExecutorDecorator<>(executor);
@@ -80,8 +77,10 @@ public class RemoteCacheSessionListener<K, V extends SessionEntity>  {
         if (shouldUpdateLocalCache(event.getType(), key, event.isCommandRetried())) {
             this.executor.submit(event, () -> {
 
-                // Should load it from remoteStore
+                // Doesn't work due https://issues.jboss.org/browse/ISPN-9323. Needs to explicitly retrieve and create it
-                cache.get(key);
+                //cache.get(key);
+
+                createRemoteEntityInCache(key, event.getVersion());
 
             });
         }
@@ -102,9 +101,30 @@ public class RemoteCacheSessionListener<K, V extends SessionEntity>  {
         }
     }
 
-    private static final int MAXIMUM_REPLACE_RETRIES = 10;
 
-    private void replaceRemoteEntityInCache(K key, long eventVersion) {
+    protected void createRemoteEntityInCache(K key, long eventVersion) {
+        VersionedValue<SessionEntityWrapper<V>> remoteSessionVersioned = remoteCache.getWithMetadata(key);
+
+        // Maybe can happen under some circumstances that remoteCache doesn't yet contain the value sent in the event (maybe just theoretically...)
+        if (remoteSessionVersioned == null || remoteSessionVersioned.getValue() == null) {
+            logger.debugf("Entity '%s' not present in remoteCache. Ignoring create",
+                    key.toString());
+            return;
+        }
+
+
+        V remoteSession = remoteSessionVersioned.getValue().getEntity();
+        SessionEntityWrapper<V> newWrapper = new SessionEntityWrapper<>(remoteSession);
+
+        logger.debugf("Read session entity wrapper from the remote cache: %s", remoteSession.toString());
+
+        // Using putIfAbsent. Theoretic possibility that entity was already put to cache by someone else
+        cache.getAdvancedCache().withFlags(Flag.SKIP_CACHE_STORE, Flag.SKIP_CACHE_LOAD, Flag.IGNORE_RETURN_VALUES)
+                .putIfAbsent(key, newWrapper);
+    }
+
+
+    protected void replaceRemoteEntityInCache(K key, long eventVersion) {
         // TODO can be optimized and remoteSession sent in the event itself?
         boolean replaced = false;
         int replaceRetries = 0;
@@ -113,7 +133,7 @@ public class RemoteCacheSessionListener<K, V extends SessionEntity>  {
             replaceRetries++;
             
             SessionEntityWrapper<V> localEntityWrapper = cache.get(key);
-            VersionedValue<SessionEntityWrapper<V>> remoteSessionVersioned = remoteCache.getVersioned(key);
+            VersionedValue<SessionEntityWrapper<V>> remoteSessionVersioned = remoteCache.getWithMetadata(key);
 
             // Probably already removed
             if (remoteSessionVersioned == null || remoteSessionVersioned.getValue() == null) {
@@ -177,11 +197,10 @@ public class RemoteCacheSessionListener<K, V extends SessionEntity>  {
             return false;
         }
 
-        if (!distributed || commandRetried) {
+        if (commandRetried) {
             result = true;
         } else {
-            String keyAddress = InfinispanUtil.getKeyPrimaryOwnerAddress(cache, key);
+            result = topologyInfo.amIOwner(cache, key);
-            result = myAddress.equals(keyAddress);
         }
 
         logger.debugf("Received event from remote store. Event '%s', key '%s', skip '%b'", type.toString(), key, !result);

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/RemoteCacheSessionsLoader.java

@@ -19,127 +19,138 @@ package org.keycloak.models.sessions.infinispan.remotestore;
 
 import java.io.Serializable;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
 import org.infinispan.Cache;
 import org.infinispan.client.hotrod.RemoteCache;
+import org.infinispan.client.hotrod.impl.RemoteCacheImpl;
+import org.infinispan.client.hotrod.impl.operations.IterationStartOperation;
+import org.infinispan.client.hotrod.impl.operations.IterationStartResponse;
+import org.infinispan.client.hotrod.impl.operations.OperationsFactory;
 import org.infinispan.commons.marshall.Marshaller;
+import org.infinispan.commons.util.CloseableIterator;
 import org.infinispan.context.Flag;
 import org.jboss.logging.Logger;
 import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
 import org.keycloak.connections.infinispan.RemoteCacheProvider;
 import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.KeycloakSessionFactory;
 import org.keycloak.models.sessions.infinispan.changes.SessionEntityWrapper;
 import org.keycloak.models.sessions.infinispan.initializer.BaseCacheInitializer;
 import org.keycloak.models.sessions.infinispan.initializer.OfflinePersistentUserSessionLoader;
 import org.keycloak.models.sessions.infinispan.initializer.SessionLoader;
 
+import static org.infinispan.client.hotrod.impl.Util.await;
+
 /**
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
-public class RemoteCacheSessionsLoader implements SessionLoader {
+public class RemoteCacheSessionsLoader implements SessionLoader<RemoteCacheSessionsLoaderContext>, Serializable {
 
     private static final Logger log = Logger.getLogger(RemoteCacheSessionsLoader.class);
 
-
-    // Javascript to be executed on remote infinispan server.
-    // Flag CACHE_MODE_LOCAL is optimization used just when remoteCache is replicated as all the entries are available locally. For distributed caches, it can't be used
-    private static final String REMOTE_SCRIPT_FOR_LOAD_SESSIONS =
-            "function loadSessions() {" +
-                    "  var flagClazz = cache.getClass().getClassLoader().loadClass(\"org.infinispan.context.Flag\"); \n" +
-                    "  var localFlag = java.lang.Enum.valueOf(flagClazz, \"CACHE_MODE_LOCAL\"); \n" +
-                    "  var cacheMode = cache.getCacheConfiguration().clustering().cacheMode(); \n" +
-                    "  var canUseLocalFlag = !cacheMode.isClustered() || cacheMode.isReplicated(); \n" +
-
-                    "  var cacheStream; \n" +
-                    "  if (canUseLocalFlag) { \n" +
-                    "      cacheStream = cache.getAdvancedCache().withFlags([ localFlag ]).entrySet().stream();\n" +
-                    "  } else { \n" +
-                    "      cacheStream = cache.getAdvancedCache().withFlags([ ]).entrySet().stream();\n" +
-                    "  }; \n" +
-
-                    "  var result = cacheStream.skip(first).limit(max).collect(java.util.stream.Collectors.toMap(\n" +
-                    "    new java.util.function.Function() {\n" +
-                    "      apply: function(entry) {\n" +
-                    "        return entry.getKey();\n" +
-                    "      }\n" +
-                    "    },\n" +
-                    "    new java.util.function.Function() {\n" +
-                    "      apply: function(entry) {\n" +
-                    "        return entry.getValue();\n" +
-                    "      }\n" +
-                    "    }\n" +
-                    "  ));\n" +
-                    "\n" +
-                    "  cacheStream.close();\n" +
-                    "  return result;\n" +
-                    "};\n" +
-                    "\n" +
-                    "loadSessions();";
-
-
-
     private final String cacheName;
+    private final int sessionsPerSegment;
 
-    public RemoteCacheSessionsLoader(String cacheName) {
+    public RemoteCacheSessionsLoader(String cacheName, int sessionsPerSegment) {
         this.cacheName = cacheName;
+        this.sessionsPerSegment = sessionsPerSegment;
     }
 
+
     @Override
     public void init(KeycloakSession session) {
+    }
+
+
+    @Override
+    public RemoteCacheSessionsLoaderContext computeLoaderContext(KeycloakSession session) {
         RemoteCache remoteCache = getRemoteCache(session);
+        int sessionsTotal = remoteCache.size();
+        int ispnSegments = getIspnSegmentsCount(remoteCache);
 
-        RemoteCache<String, String> scriptCache = remoteCache.getRemoteCacheManager().getCache(RemoteCacheProvider.SCRIPT_CACHE_NAME);
+        return new RemoteCacheSessionsLoaderContext(ispnSegments, sessionsPerSegment, sessionsTotal);
 
-        if (!scriptCache.containsKey("load-sessions.js")) {
+    }
-            scriptCache.put("load-sessions.js",
+
-                    "// mode=local,language=javascript\n" +
+
-                            REMOTE_SCRIPT_FOR_LOAD_SESSIONS);
+    protected int getIspnSegmentsCount(RemoteCache remoteCache) {
+        OperationsFactory operationsFactory = ((RemoteCacheImpl) remoteCache).getOperationsFactory();
+
+        // Same like RemoteCloseableIterator.startInternal
+        IterationStartOperation iterationStartOperation = operationsFactory.newIterationStartOperation(null, null, null, sessionsPerSegment, false);
+        IterationStartResponse startResponse = await(iterationStartOperation.execute());
+
+        try {
+            // Could happen for non-clustered caches
+            if (startResponse.getSegmentConsistentHash() == null) {
+                return -1;
+            } else {
+                return startResponse.getSegmentConsistentHash().getNumSegments();
+            }
+        } finally {
+            startResponse.getChannel().close();
         }
     }
 
-    @Override
-    public int getSessionsCount(KeycloakSession session) {
-        RemoteCache remoteCache = getRemoteCache(session);
-        return remoteCache.size();
-    }
 
     @Override
-    public boolean loadSessions(KeycloakSession session, int first, int max) {
+    public boolean loadSessions(KeycloakSession session, RemoteCacheSessionsLoaderContext context, int segment) {
         Cache cache = getCache(session);
         Cache decoratedCache = cache.getAdvancedCache().withFlags(Flag.SKIP_CACHE_LOAD, Flag.SKIP_CACHE_STORE, Flag.IGNORE_RETURN_VALUES);
+        RemoteCache remoteCache = getRemoteCache(session);
 
-        RemoteCache<?, ?> remoteCache = getRemoteCache(session);
+        Set<Integer> myIspnSegments = getMyIspnSegments(segment, context);
 
-        log.debugf("Will do bulk load of sessions from remote cache '%s' . First: %d, max: %d", cache.getName(), first, max);
+        log.debugf("Will do bulk load of sessions from remote cache '%s' . Segment: %d", cache.getName(), segment);
 
-        Map<String, Integer> remoteParams = new HashMap<>();
+        CloseableIterator<Map.Entry> iterator = null;
-        remoteParams.put("first", first);
+        int countLoaded = 0;
-        remoteParams.put("max", max);
+        try {
-        Map<byte[], byte[]> remoteObjects = remoteCache.execute("load-sessions.js", remoteParams);
+            iterator = remoteCache.retrieveEntries(null, myIspnSegments, context.getSessionsPerSegment());
-
+            while (iterator.hasNext()) {
-        log.debugf("Successfully finished loading sessions '%s' . First: %d, max: %d", cache.getName(), first, max);
+                countLoaded++;
-
+                Map.Entry entry = iterator.next();
-        Marshaller marshaller = remoteCache.getRemoteCacheManager().getMarshaller();
+                decoratedCache.putAsync(entry.getKey(), entry.getValue());
-
+            }
-        for (Map.Entry<byte[], byte[]> entry : remoteObjects.entrySet()) {
+        } catch (RuntimeException e) {
-            try {
+            log.warnf(e, "Error loading sessions from remote cache '%s' for segment '%d'", remoteCache.getName(), segment);
-                Object key = marshaller.objectFromByteBuffer(entry.getKey());
+            throw e;
-                SessionEntityWrapper entityWrapper = (SessionEntityWrapper) marshaller.objectFromByteBuffer(entry.getValue());
+        } finally {
-
+            if (iterator != null) {
-                decoratedCache.putAsync(key, entityWrapper);
+                iterator.close();
-            } catch (Exception e) {
-                log.warn("Error loading session from remote cache", e);
             }
         }
 
+        log.debugf("Successfully finished loading sessions from cache '%s' . Segment: %d, Count of sessions loaded: %d", cache.getName(), segment, countLoaded);
+
         return true;
     }
 
 
-    private Cache getCache(KeycloakSession session) {
+    // Compute set of ISPN segments into 1 "worker" segment
-        InfinispanConnectionProvider ispn = session.getProvider(InfinispanConnectionProvider.class);
+    protected Set<Integer> getMyIspnSegments(int segment, RemoteCacheSessionsLoaderContext ctx) {
-        return ispn.getCache(cacheName);
+        // Remote cache is non-clustered
+        if (ctx.getIspnSegmentsCount() < 0) {
+            return null;
+        }
+
+        if (ctx.getIspnSegmentsCount() % ctx.getSegmentsCount() > 0) {
+            throw new IllegalStateException("Illegal state. IspnSegmentsCount: " + ctx.getIspnSegmentsCount() + ", segmentsCount: " + ctx.getSegmentsCount());
+        }
+
+        int countPerSegment = ctx.getIspnSegmentsCount() / ctx.getSegmentsCount();
+        int first = segment * countPerSegment;
+        int last = first + countPerSegment - 1;
+
+        Set<Integer> myIspnSegments = new HashSet<>();
+        for (int i=first ; i<=last ; i++) {
+            myIspnSegments.add(i);
+        }
+        return myIspnSegments;
+
     }
 
 
@@ -165,12 +176,28 @@ public class RemoteCacheSessionsLoader implements SessionLoader {
 
     @Override
     public void afterAllSessionsLoaded(BaseCacheInitializer initializer) {
-
     }
 
+
+    protected Cache getCache(KeycloakSession session) {
+        InfinispanConnectionProvider ispn = session.getProvider(InfinispanConnectionProvider.class);
+        return ispn.getCache(cacheName);
+    }
+
+
     // Get remoteCache, which may be secured
-    private RemoteCache getRemoteCache(KeycloakSession session) {
+    protected RemoteCache getRemoteCache(KeycloakSession session) {
         InfinispanConnectionProvider ispn = session.getProvider(InfinispanConnectionProvider.class);
         return ispn.getRemoteCache(cacheName);
     }
+
+
+    @Override
+    public String toString() {
+        return new StringBuilder("RemoteCacheSessionsLoader [ ")
+                .append("cacheName: ").append(cacheName)
+                .append(", sessionsPerSegment: ").append(sessionsPerSegment)
+                .append(" ]")
+                .toString();
+    }
 }

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/RemoteCacheSessionsLoaderContext.java

@@ -0,0 +1,99 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.models.sessions.infinispan.remotestore;
+
+import java.io.Serializable;
+
+import org.keycloak.models.sessions.infinispan.initializer.SessionLoader;
+
+/**
+ *
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class RemoteCacheSessionsLoaderContext implements SessionLoader.LoaderContext, Serializable {
+
+    // Count of hash segments for remote infinispan cache. It's by default 256 for distributed/replicated caches
+    private final int ispnSegmentsCount;
+
+    // Count of segments (worker iterations for distributedExecutionService executions on KC side). Each segment will be 1 worker iteration.
+    // Count of segments could be lower than "ispnSegmentsCount" and depends on the size of the cache. For example if we have cache with just 500 items,
+    // we don't need 256 segments and send 256 requests to remoteCache to preload thing. Instead, we will have lower number of segments (EG. 8)
+    // and we will map more ispnSegments into 1 worker segment (In this case 256 / 8 = 32. So 32 ISPN segments mapped to each worker segment)
+    private final int segmentsCount;
+
+    private final int sessionsPerSegment;
+    private final int sessionsTotal;
+
+
+    public RemoteCacheSessionsLoaderContext(int ispnSegmentsCount, int sessionsPerSegment, int sessionsTotal) {
+        this.ispnSegmentsCount = ispnSegmentsCount;
+        this.sessionsPerSegment = sessionsPerSegment;
+        this.sessionsTotal = sessionsTotal;
+        this.segmentsCount = computeSegmentsCount(sessionsTotal, sessionsPerSegment, ispnSegmentsCount);
+    }
+
+
+    private int computeSegmentsCount(int sessionsTotal, int sessionsPerSegment, int ispnSegments) {
+        // No support by remote ISPN cache for segments. This can happen if remoteCache is local (non-clustered)
+        if (ispnSegments < 0) {
+            return 1;
+        }
+
+        int seg = sessionsTotal / sessionsPerSegment;
+        if (sessionsTotal % sessionsPerSegment > 0) {
+            seg = seg + 1;
+        }
+
+        int seg2 = 1;
+        while (seg2<seg && seg2<ispnSegments) {
+            seg2 = seg2 << 1;
+        }
+
+        return seg2;
+    }
+
+
+    @Override
+    public int getSegmentsCount() {
+        return segmentsCount;
+    }
+
+    public int getIspnSegmentsCount() {
+        return ispnSegmentsCount;
+    }
+
+    public int getSessionsPerSegment() {
+        return sessionsPerSegment;
+    }
+
+    public int getSessionsTotal() {
+        return sessionsTotal;
+    }
+
+
+    @Override
+    public String toString() {
+        return new StringBuilder("RemoteCacheSessionsLoaderContext [ ")
+                .append("segmentsCount: ").append(segmentsCount)
+                .append(", ispnSegmentsCount: ").append(ispnSegmentsCount)
+                .append(", sessionsPerSegment: ").append(sessionsPerSegment)
+                .append(", sessionsTotal: ").append(sessionsTotal)
+                .append(" ]")
+                .toString();
+    }
+}

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/stream/Mappers.java

@@ -17,7 +17,6 @@
 
 package org.keycloak.models.sessions.infinispan.stream;
 
-import org.infinispan.stream.SerializableSupplier;
 import org.keycloak.models.sessions.infinispan.changes.SessionEntityWrapper;
 import org.keycloak.models.sessions.infinispan.entities.AuthenticatedClientSessionEntity;
 import org.keycloak.models.sessions.infinispan.entities.LoginFailureEntity;

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/util/InfinispanUtil.java

@@ -27,6 +27,7 @@ import org.infinispan.persistence.manager.PersistenceManager;
 import org.infinispan.persistence.remote.RemoteStore;
 import org.infinispan.remoting.transport.Transport;
 import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
+import org.keycloak.connections.infinispan.TopologyInfo;
 import org.keycloak.models.KeycloakSession;
 
 /**
@@ -52,34 +53,8 @@ public class InfinispanUtil {
     }
 
 
-    public static boolean isDistributedCache(Cache ispnCache) {
+    public static TopologyInfo getTopologyInfo(KeycloakSession session) {
-        CacheMode cacheMode = ispnCache.getCacheConfiguration().clustering().cacheMode();
+        return session.getProvider(InfinispanConnectionProvider.class).getTopologyInfo();
-        return cacheMode.isDistributed();
-    }
-
-
-    public static String getMyAddress(KeycloakSession session) {
-        return session.getProvider(InfinispanConnectionProvider.class).getNodeName();
-    }
-
-    public static String getMySite(KeycloakSession session) {
-        return session.getProvider(InfinispanConnectionProvider.class).getSiteName();
-    }
-
-
-    /**
-     *
-     * @param ispnCache
-     * @param key
-     * @return address of the node, who is owner of the specified key in current cluster
-     */
-    public static String getKeyPrimaryOwnerAddress(Cache ispnCache, Object key) {
-        DistributionManager distManager = ispnCache.getAdvancedCache().getDistributionManager();
-        if (distManager == null) {
-            throw new IllegalArgumentException("Cache '" + ispnCache.getName() + "' is not distributed cache");
-        }
-
-        return distManager.getPrimaryLocation(key).toString();
     }
 
 

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/util/KeycloakMarshallUtil.java

@@ -165,7 +165,6 @@ public class KeycloakMarshallUtil {
         return isSet ? input.readInt() : null;
     }
 
-
     public static class ConcurrentHashMapBuilder<K, V> implements MarshallUtil.MapBuilder<K, V, ConcurrentHashMap<K, V>> {
 
         @Override

model/infinispan/src/test/java/org/keycloak/cluster/infinispan/ConcurrencyJDGCachePutTest.java

@@ -44,7 +44,7 @@ import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
  *
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
-public class ConcurrencyJDGRemoteCacheTest {
+public class ConcurrencyJDGCachePutTest {
 
     private static Map<String, EntryInfo> state = new HashMap<>();
 
@@ -54,7 +54,7 @@ public class ConcurrencyJDGRemoteCacheTest {
 
     public static void main(String[] args) throws Exception {
         // Init map somehow
-        for (int i=0 ; i<3000 ; i++) {
+        for (int i=0 ; i<1000 ; i++) {
             String key = "key-" + i;
             state.put(key, new EntryInfo());
         }
@@ -74,12 +74,24 @@ public class ConcurrencyJDGRemoteCacheTest {
 
         long took = System.currentTimeMillis() - start;
 
+        Map<String, EntryInfo> failedState = new HashMap<>();
+
         // Output
         for (Map.Entry<String, EntryInfo> entry : state.entrySet()) {
             System.out.println(entry.getKey() + ":::" + entry.getValue());
+
+            if (entry.getValue().th1.get() != entry.getValue().th2.get()) {
+                failedState.put(entry.getKey(), entry.getValue());
+            }
+
             worker1.cache.remove(entry.getKey());
         }
 
+        System.out.println("\nFAILED ENTRIES. SIZE: " + failedState.size() + "\n");
+        for (Map.Entry<String, EntryInfo> entry : failedState.entrySet()) {
+            System.out.println(entry.getKey() + ":::" + entry.getValue());
+        }
+
         System.out.println("Took: " + took + " ms");
 
         // Finish JVM

model/infinispan/src/test/java/org/keycloak/cluster/infinispan/ConcurrencyJDGCacheReplaceTest.java

@@ -51,9 +51,9 @@ import org.infinispan.persistence.remote.configuration.RemoteStoreConfigurationB
  *
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
-public class ConcurrencyJDGSessionsCacheTest {
+public class ConcurrencyJDGCacheReplaceTest {
 
-    protected static final Logger logger = Logger.getLogger(ConcurrencyJDGSessionsCacheTest.class);
+    protected static final Logger logger = Logger.getLogger(ConcurrencyJDGCacheReplaceTest.class);
 
     private static final int ITERATION_PER_WORKER = 1000;
 
@@ -152,6 +152,8 @@ public class ConcurrencyJDGSessionsCacheTest {
         InfinispanUtil.getRemoteCache(cache2).replace("123", session);
 
         // Create caches, listeners and finally worker threads
+        remoteCache1 = InfinispanUtil.getRemoteCache(cache1);
+        remoteCache2 = InfinispanUtil.getRemoteCache(cache2);
         Thread worker1 = createWorker(cache1, 1);
         Thread worker2 = createWorker(cache2, 2);
 
@@ -256,7 +258,7 @@ public class ConcurrencyJDGSessionsCacheTest {
 
             executor.submit(() -> {
                 // TODO: can be optimized - object sent in the event
-                VersionedValue<SessionEntity> versionedVal = remoteCache.getVersioned(cacheKey);
+                VersionedValue<SessionEntity> versionedVal = remoteCache.getWithMetadata(cacheKey);
                 for (int i = 0; i < 10; i++) {
 
                     if (versionedVal.getVersion() < event.getVersion()) {
@@ -267,7 +269,7 @@ public class ConcurrencyJDGSessionsCacheTest {
                             throw new RuntimeException(ie);
                         }
 
-                        versionedVal = remoteCache.getVersioned(cacheKey);
+                        versionedVal = remoteCache.getWithMetadata(cacheKey);
                     } else {
                         break;
                     }
@@ -316,7 +318,7 @@ public class ConcurrencyJDGSessionsCacheTest {
 
                 ReplaceStatus replaced = ReplaceStatus.NOT_REPLACED;
                 while (replaced != ReplaceStatus.REPLACED) {
-                    VersionedValue<UserSessionEntity> versioned = remoteCache.getVersioned("123");
+                    VersionedValue<UserSessionEntity> versioned = remoteCache.getWithMetadata("123");
                     UserSessionEntity oldSession = versioned.getValue();
                     //UserSessionEntity clone = DistributedCacheConcurrentWritesTest.cloneSession(oldSession);
                     UserSessionEntity clone = oldSession;
@@ -340,7 +342,7 @@ public class ConcurrencyJDGSessionsCacheTest {
 
                 // Try to see if remoteCache on 2nd DC is immediatelly seeing our change
                 RemoteCache secondDCRemoteCache = myThreadId == 1 ? remoteCache2 : remoteCache1;
-                UserSessionEntity thatSession = (UserSessionEntity) secondDCRemoteCache.get("123");
+                //UserSessionEntity thatSession = (UserSessionEntity) secondDCRemoteCache.get("123");
 
                 //Assert.assertEquals("someVal", thatSession.getNotes().get(noteKey));
                 //System.out.println("Passed");

model/infinispan/src/test/java/org/keycloak/cluster/infinispan/ConcurrencyJDGRemoteCacheClientListenersTest.java

@@ -19,6 +19,8 @@ package org.keycloak.cluster.infinispan;
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.infinispan.Cache;
@@ -41,7 +43,7 @@ import org.keycloak.models.sessions.infinispan.util.InfinispanUtil;
  * Test that hotrod ClientListeners are correctly executed as expected
  *
  * STEPS TO REPRODUCE:
- * - Unzip infinispan-server-8.2.6.Final to some locations ISPN1 and ISPN2
+ * - Unzip infinispan-server-9.2.4.Final to some locations ISPN1 and ISPN2
  *
  * - Edit both ISPN1/standalone/configuration/clustered.xml and ISPN2/standalone/configuration/clustered.xml . Configure cache in container "clustered"
  *
@@ -55,7 +57,7 @@ import org.keycloak.models.sessions.infinispan.util.InfinispanUtil;
  ./standalone.sh -c clustered.xml -Djava.net.preferIPv4Stack=true -Djboss.socket.binding.port-offset=1010 -Djboss.default.multicast.address=234.56.78.99 -Djboss.node.name=cache-server
 
     - Run server2
- ./standalone.sh -c clustered.xml -Djava.net.preferIPv4Stack=true -Djboss.socket.binding.port-offset=2010 -Djboss.default.multicast.address=234.56.78.99 -Djboss.node.name=cache-server-dc-2
+ ./standalone.sh -c clustered.xml -Djava.net.preferIPv4Stack=true -Djboss.socket.binding.port-offset=2010 -Djboss.default.multicast.address=234.56.78.100 -Djboss.node.name=cache-server-dc-2
 
     - Run this test as main class from IDE
  *
@@ -145,10 +147,12 @@ public class ConcurrencyJDGRemoteCacheClientListenersTest {
 
         private final RemoteCache<String, Integer> remoteCache;
         private final int threadId;
+        private Executor executor;
 
         public HotRodListener(Cache<String, Integer> cache, int threadId) {
             this.remoteCache = InfinispanUtil.getRemoteCache(cache);
             this.threadId = threadId;
+            this.executor = Executors.newCachedThreadPool();
         }
 
         //private AtomicInteger listenerCount = new AtomicInteger(0);
@@ -156,7 +160,12 @@ public class ConcurrencyJDGRemoteCacheClientListenersTest {
         @ClientCacheEntryCreated
         public void created(ClientCacheEntryCreatedEvent event) {
             String cacheKey = (String) event.getKey();
-            event(cacheKey, event.getVersion(), true);
+
+            executor.execute(() -> {
+
+                event(cacheKey, event.getVersion(), true);
+
+            });
 
         }
 
@@ -164,7 +173,11 @@ public class ConcurrencyJDGRemoteCacheClientListenersTest {
         @ClientCacheEntryModified
         public void updated(ClientCacheEntryModifiedEvent event) {
             String cacheKey = (String) event.getKey();
-            event(cacheKey, event.getVersion(), false);
+            executor.execute(() -> {
+
+                event(cacheKey, event.getVersion(), false);
+
+            });
         }
 
 
@@ -174,7 +187,7 @@ public class ConcurrencyJDGRemoteCacheClientListenersTest {
 
             totalListenerCalls.incrementAndGet();
 
-            VersionedValue<Integer> versionedVal = remoteCache.getVersioned(cacheKey);
+            VersionedValue<Integer> versionedVal = remoteCache.getWithMetadata(cacheKey);
 
             if (versionedVal.getVersion() < version) {
                 System.err.println("INCOMPATIBLE VERSION. event version: " + version + ", entity version: " + versionedVal.getVersion());

model/infinispan/src/test/java/org/keycloak/cluster/infinispan/RemoteCacheSessionsLoaderTest.java

@@ -0,0 +1,160 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.cluster.infinispan;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.infinispan.Cache;
+import org.infinispan.client.hotrod.RemoteCache;
+import org.infinispan.manager.EmbeddedCacheManager;
+import org.infinispan.persistence.remote.configuration.RemoteStoreConfigurationBuilder;
+import org.jboss.logging.Logger;
+import org.junit.Assert;
+import org.keycloak.common.util.Time;
+import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.sessions.infinispan.changes.SessionEntityWrapper;
+import org.keycloak.models.sessions.infinispan.entities.UserSessionEntity;
+import org.keycloak.models.sessions.infinispan.remotestore.RemoteCacheSessionsLoader;
+import org.keycloak.models.sessions.infinispan.remotestore.RemoteCacheSessionsLoaderContext;
+import org.keycloak.models.sessions.infinispan.util.InfinispanUtil;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class RemoteCacheSessionsLoaderTest {
+
+    protected static final Logger logger = Logger.getLogger(RemoteCacheSessionsLoaderTest.class);
+
+    private static final int COUNT = 10000;
+
+    public static void main(String[] args) throws Exception {
+        String cacheName = InfinispanConnectionProvider.USER_SESSION_CACHE_NAME;
+        Cache cache1 = createManager(1, cacheName).getCache(cacheName);
+        Cache cache2 = cache1.getCacheManager().getCache("local");
+        RemoteCache remoteCache = InfinispanUtil.getRemoteCache(cache1);
+        cache1.clear();
+        cache2.clear();
+        remoteCache.clear();
+
+        try {
+
+            for (int i=0 ; i<COUNT ; i++) {
+                // Create initial item
+                UserSessionEntity session = new UserSessionEntity();
+                session.setId("loader-key-" + i);
+                session.setRealmId("master");
+                session.setBrokerSessionId("!23123123");
+                session.setBrokerUserId(null);
+                session.setUser("admin");
+                session.setLoginUsername("admin");
+                session.setIpAddress("123.44.143.178");
+                session.setStarted(Time.currentTime());
+                session.setLastSessionRefresh(Time.currentTime());
+
+                SessionEntityWrapper<UserSessionEntity> wrappedSession = new SessionEntityWrapper<>(session);
+
+                // Create caches, listeners and finally worker threads
+                remoteCache.put("loader-key-" + i, wrappedSession);
+                Assert.assertFalse(cache2.containsKey("loader-key-" + i));
+
+                if (i % 1000 == 0) {
+                    logger.infof("%d sessions added", i);
+                }
+            }
+
+
+//            RemoteCacheSessionsLoader loader = new RemoteCacheSessionsLoader(InfinispanConnectionProvider.USER_SESSION_CACHE_NAME, 64) {
+//
+//                @Override
+//                protected Cache getCache(KeycloakSession session) {
+//                    return cache2;
+//                }
+//
+//                @Override
+//                protected RemoteCache getRemoteCache(KeycloakSession session) {
+//                    return remoteCache;
+//                }
+//
+//            };
+
+            // Just to be able to test serializability
+            RemoteCacheSessionsLoader loader = new CustomLoader(cacheName, 64, cache2, remoteCache);
+
+            loader.init(null);
+            RemoteCacheSessionsLoaderContext ctx = loader.computeLoaderContext(null);
+            Assert.assertEquals(ctx.getSessionsTotal(), COUNT);
+            Assert.assertEquals(ctx.getIspnSegmentsCount(), 256);
+            //Assert.assertEquals(ctx.getSegmentsCount(), 16);
+            Assert.assertEquals(ctx.getSessionsPerSegment(), 64);
+
+            int totalCount = 0;
+            logger.infof("segmentsCount: %d", ctx.getSegmentsCount());
+
+            Set<String> visitedKeys = new HashSet<>();
+            for (int currentSegment=0 ; currentSegment<ctx.getSegmentsCount() ; currentSegment++) {
+                logger.infof("Loading segment %d", currentSegment);
+                loader.loadSessions(null, ctx, currentSegment);
+
+                logger.infof("Loaded %d keys for segment %d", cache2.keySet().size(), currentSegment);
+                totalCount = totalCount + cache2.keySet().size();
+                visitedKeys.addAll(cache2.keySet());
+                cache2.clear();
+            }
+
+            Assert.assertEquals(totalCount, COUNT);
+            Assert.assertEquals(visitedKeys.size(), COUNT);
+            logger.infof("SUCCESS: Loaded %d sessions", totalCount);
+        } finally {
+            // Finish JVM
+            cache1.getCacheManager().stop();
+        }
+    }
+
+
+    private static EmbeddedCacheManager createManager(int threadId, String cacheName) {
+        return new TestCacheManagerFactory().createManager(threadId, cacheName, RemoteStoreConfigurationBuilder.class);
+    }
+
+
+    public static class CustomLoader extends RemoteCacheSessionsLoader {
+
+        private final transient Cache cache2;
+        private final transient RemoteCache remoteCache;
+
+        public CustomLoader(String cacheName, int sessionsPerSegment, Cache cache2, RemoteCache remoteCache) {
+            super(cacheName, sessionsPerSegment);
+            this.cache2 = cache2;
+            this.remoteCache = remoteCache;
+
+        }
+
+        @Override
+        protected Cache getCache(KeycloakSession session) {
+            return cache2;
+        }
+
+        @Override
+        protected RemoteCache getRemoteCache(KeycloakSession session) {
+            return remoteCache;
+        }
+
+    }
+
+}

model/infinispan/src/test/java/org/keycloak/cluster/infinispan/TestCacheManagerFactory.java

@@ -53,6 +53,7 @@ class TestCacheManagerFactory {
         Configuration invalidationCacheConfiguration = getCacheBackedByRemoteStore(threadId, cacheName, builderClass);
 
         cacheManager.defineConfiguration(cacheName, invalidationCacheConfiguration);
+        cacheManager.defineConfiguration("local", new ConfigurationBuilder().build());
         return cacheManager;
 
     }
@@ -75,6 +76,8 @@ class TestCacheManagerFactory {
                 .rawValues(true)
                 .forceReturnValues(false)
                 .marshaller(KeycloakHotRodMarshallerFactory.class.getName())
+                //.protocolVersion(ProtocolVersion.PROTOCOL_VERSION_26)
+                //.maxBatchSize(5)
                 .addServer()
                     .host(host)
                     .port(port)

model/infinispan/src/test/java/org/keycloak/keys/infinispan/InfinispanKeyStorageProviderTest.java

@@ -161,7 +161,10 @@ public class InfinispanKeyStorageProviderTest {
         final DefaultCacheManager cacheManager = new DefaultCacheManager(gcb.build());
 
         ConfigurationBuilder cb = new ConfigurationBuilder();
-        cb.eviction().strategy(EvictionStrategy.LRU).type(EvictionType.COUNT).size(InfinispanConnectionProvider.KEYS_CACHE_DEFAULT_MAX);
+        cb.memory()
+                .evictionStrategy(EvictionStrategy.REMOVE)
+                .evictionType(EvictionType.COUNT)
+                .size(InfinispanConnectionProvider.KEYS_CACHE_DEFAULT_MAX);
         cb.jmxStatistics().enabled(true);
         Configuration cfg = cb.build();
 

model/infinispan/src/test/java/org/keycloak/models/sessions/infinispan/initializer/ConcurrencyLockingTest.java

@@ -6,7 +6,7 @@ import org.infinispan.configuration.cache.ConfigurationBuilder;
 import org.infinispan.configuration.global.GlobalConfigurationBuilder;
 import org.infinispan.manager.DefaultCacheManager;
 import org.infinispan.transaction.LockingMode;
-import org.infinispan.transaction.lookup.DummyTransactionManagerLookup;
+import org.infinispan.transaction.lookup.EmbeddedTransactionManagerLookup;
 import org.junit.Ignore;
 import org.junit.Test;
 import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
@@ -71,7 +71,7 @@ public class ConcurrencyLockingTest {
 
         ConfigurationBuilder counterConfigBuilder = new ConfigurationBuilder();
         counterConfigBuilder.invocationBatching().enable();
-        counterConfigBuilder.transaction().transactionManagerLookup(new DummyTransactionManagerLookup());
+        counterConfigBuilder.transaction().transactionManagerLookup(new EmbeddedTransactionManagerLookup());
         counterConfigBuilder.transaction().lockingMode(LockingMode.PESSIMISTIC);
         Configuration counterCacheConfiguration = counterConfigBuilder.build();
 

model/infinispan/src/test/java/org/keycloak/models/sessions/infinispan/initializer/ConcurrencyVersioningTest.java

@@ -9,7 +9,7 @@ import org.infinispan.configuration.global.GlobalConfigurationBuilder;
 import org.infinispan.manager.DefaultCacheManager;
 import org.infinispan.manager.EmbeddedCacheManager;
 import org.infinispan.transaction.TransactionMode;
-import org.infinispan.transaction.lookup.DummyTransactionManagerLookup;
+import org.infinispan.transaction.lookup.EmbeddedTransactionManagerLookup;
 import org.infinispan.util.concurrent.IsolationLevel;
 import org.junit.Assert;
 import org.junit.Ignore;
@@ -251,7 +251,7 @@ public class ConcurrencyVersioningTest {
         invalidationConfigBuilder
                 //.invocationBatching().enable()
                 .transaction().transactionMode(TransactionMode.TRANSACTIONAL)
-                .transaction().transactionManagerLookup(new DummyTransactionManagerLookup())
+                .transaction().transactionManagerLookup(new EmbeddedTransactionManagerLookup())
                 .locking().isolationLevel(IsolationLevel.REPEATABLE_READ).writeSkewCheck(true).versioning().enable().scheme(VersioningScheme.SIMPLE);
 
 

model/infinispan/src/test/java/org/keycloak/models/sessions/infinispan/initializer/DistributedCacheWriteSkewTest.java

@@ -30,7 +30,7 @@ import org.infinispan.manager.DefaultCacheManager;
 import org.infinispan.manager.EmbeddedCacheManager;
 import org.infinispan.remoting.transport.jgroups.JGroupsTransport;
 import org.infinispan.transaction.LockingMode;
-import org.infinispan.transaction.lookup.DummyTransactionManagerLookup;
+import org.infinispan.transaction.lookup.EmbeddedTransactionManagerLookup;
 import org.infinispan.util.concurrent.IsolationLevel;
 import org.jgroups.JChannel;
 import org.keycloak.common.util.Time;
@@ -202,7 +202,7 @@ public class DistributedCacheWriteSkewTest {
 
            // distConfigBuilder.invocationBatching().enable();
             //distConfigBuilder.transaction().transactionMode(TransactionMode.TRANSACTIONAL);
-            distConfigBuilder.transaction().transactionManagerLookup(new DummyTransactionManagerLookup());
+            distConfigBuilder.transaction().transactionManagerLookup(new EmbeddedTransactionManagerLookup());
             distConfigBuilder.transaction().lockingMode(LockingMode.OPTIMISTIC);
         }
         Configuration distConfig = distConfigBuilder.build();

model/infinispan/src/test/java/org/keycloak/models/sessions/infinispan/initializer/InitializerStateTest.java

@@ -20,6 +20,7 @@ package org.keycloak.models.sessions.infinispan.initializer;
 import org.junit.Assert;
 import org.junit.Test;
 import org.keycloak.models.cache.infinispan.UserCacheSession;
+import org.keycloak.models.sessions.infinispan.remotestore.RemoteCacheSessionsLoaderContext;
 import org.keycloak.storage.CacheableStorageProviderModel;
 
 import java.text.DateFormat;
@@ -32,9 +33,55 @@ import java.util.List;
  */
 public class InitializerStateTest {
 
+    @Test
+    public void testOfflineLoaderContext() {
+        OfflinePersistentUserSessionLoaderContext ctx = new OfflinePersistentUserSessionLoaderContext(28, 5);
+        Assert.assertEquals(ctx.getSegmentsCount(), 6);
+
+        ctx = new OfflinePersistentUserSessionLoaderContext(19, 5);
+        Assert.assertEquals(ctx.getSegmentsCount(), 4);
+
+        ctx = new OfflinePersistentUserSessionLoaderContext(20, 5);
+        Assert.assertEquals(ctx.getSegmentsCount(), 4);
+
+        ctx = new OfflinePersistentUserSessionLoaderContext(21, 5);
+        Assert.assertEquals(ctx.getSegmentsCount(), 5);
+    }
+
+
+    @Test
+    public void testRemoteLoaderContext() {
+        assertSegmentsForRemoteLoader(0, 64, -1, 1);
+        assertSegmentsForRemoteLoader(0, 64, 256, 1);
+        assertSegmentsForRemoteLoader(5, 64, 256, 1);
+        assertSegmentsForRemoteLoader(63, 64, 256, 1);
+        assertSegmentsForRemoteLoader(64, 64, 256, 1);
+        assertSegmentsForRemoteLoader(65, 64, 256, 2);
+        assertSegmentsForRemoteLoader(127, 64, 256, 2);
+        assertSegmentsForRemoteLoader(1000, 64, 256, 16);
+
+        assertSegmentsForRemoteLoader(2047, 64, 256, 32);
+        assertSegmentsForRemoteLoader(2048, 64, 256, 32);
+        assertSegmentsForRemoteLoader(2049, 64, 256, 64);
+
+        assertSegmentsForRemoteLoader(1000, 64, 256, 16);
+        assertSegmentsForRemoteLoader(10000, 64, 256, 256);
+        assertSegmentsForRemoteLoader(1000000, 64, 256, 256);
+        assertSegmentsForRemoteLoader(10000000, 64, 256, 256);
+    }
+
+    private void assertSegmentsForRemoteLoader(int sessionsTotal, int sessionsPerSegment, int ispnSegmentsCount, int expectedSegments) {
+        RemoteCacheSessionsLoaderContext ctx = new RemoteCacheSessionsLoaderContext(ispnSegmentsCount, sessionsPerSegment, sessionsTotal);
+        Assert.assertEquals(expectedSegments, ctx.getSegmentsCount());
+    }
+
+
     @Test
     public void testComputationState() {
-        InitializerState state = new InitializerState(28, 5);
+        OfflinePersistentUserSessionLoaderContext ctx = new OfflinePersistentUserSessionLoaderContext(28, 5);
+        Assert.assertEquals(ctx.getSegmentsCount(), 6);
+
+        InitializerState state = new InitializerState(ctx.getSegmentsCount());
 
         Assert.assertFalse(state.isFinished());
         List<Integer> segments = state.getUnfinishedSegments(3);

pom.xml

@@ -44,16 +44,18 @@
         <maven.compiler.target>1.7</maven.compiler.target>
         <maven.compiler.source>1.7</maven.compiler.source>
 
-        <wildfly.version>11.0.0.Final</wildfly.version>
+        <wildfly.version>13.0.0.Final</wildfly.version>
-        <wildfly.build-tools.version>1.2.2.Final</wildfly.build-tools.version>
+        <wildfly.build-tools.version>1.2.10.Final</wildfly.build-tools.version>
-        <eap.version>7.1.4.GA-redhat-1</eap.version>
+        <eap.version>7.2.0.CD13-redhat-4</eap.version>
-        <eap.build-tools.version>1.2.2.Final</eap.build-tools.version>
+        <eap.build-tools.version>1.2.10.Final</eap.build-tools.version>
-        <wildfly.core.version>3.0.10.Final</wildfly.core.version>
+        <wildfly.core.version>5.0.0.Final</wildfly.core.version>
         <wildfly10.core.version>2.0.10.Final</wildfly10.core.version>
 
         <jboss.as.version>7.2.0.Final</jboss.as.version>
 
-        <aesh.version>0.66.19</aesh.version>
+        <jboss.aesh.version>0.66.19</jboss.aesh.version>
+        <aesh.version>1.4</aesh.version>
+        <aesh.readline.version>1.7</aesh.readline.version>
         <apache.httpcomponents.version>4.5.2</apache.httpcomponents.version>
         <apache.httpcomponents.httpcore.version>4.4.4</apache.httpcomponents.httpcore.version>
         <apache.mime4j.version>0.6</apache.mime4j.version>
@@ -65,30 +67,30 @@
         <h2.version>1.4.193</h2.version>
         <hibernate.entitymanager.version>5.1.15.Final</hibernate.entitymanager.version>
         <hibernate.javax.persistence.version>1.0.0.Final</hibernate.javax.persistence.version>
-        <infinispan.version>8.2.11.Final</infinispan.version>
+        <infinispan.version>9.2.4.Final</infinispan.version>
         <jackson.version>2.8.11</jackson.version>
         <jackson.databind.version>2.8.11.1</jackson.databind.version>
         <javax.mail.version>1.5.6</javax.mail.version>
         <jboss.logging.version>3.3.1.Final</jboss.logging.version>
         <jboss.logging.tools.version>2.1.0.Final</jboss.logging.tools.version>
         <jboss.logging.tools.wf8.version>1.2.0.Final</jboss.logging.tools.wf8.version>
-        <jboss-jaxrs-api_2.0_spec>1.0.0.Final</jboss-jaxrs-api_2.0_spec>
+        <jboss-jaxrs-api_2.1_spec>1.0.0.Final</jboss-jaxrs-api_2.1_spec>
         <jboss-transaction-api_1.2_spec>1.0.1.Final</jboss-transaction-api_1.2_spec>
         <jboss.spec.javax.xml.bind.jboss-jaxb-api_2.2_spec.version>1.0.4.Final</jboss.spec.javax.xml.bind.jboss-jaxb-api_2.2_spec.version>
         <log4j.version>1.2.17</log4j.version>
-        <resteasy.version>3.0.26.Final</resteasy.version>
+        <resteasy.version>3.5.1.Final</resteasy.version>
         <owasp.html.sanitizer.version>20180219.1</owasp.html.sanitizer.version>
         <slf4j.version>1.7.22</slf4j.version>
         <sun.istack.version>2.21</sun.istack.version>
         <sun.jaxb.version>2.2.11</sun.jaxb.version>
         <sun.xsom.version>20140925</sun.xsom.version>
-        <undertow.version>1.4.18.Final</undertow.version>
+        <undertow.version>2.0.9.Final</undertow.version>
-        <elytron.version>1.1.10.Final</elytron.version>
+        <elytron.version>1.3.3.Final</elytron.version>
         <elytron.undertow-server.version>1.0.1.Final</elytron.undertow-server.version>
         <woodstox.version>5.0.3</woodstox.version>
         <xmlsec.version>2.0.9</xmlsec.version>
         <glassfish.json.version>1.0.4</glassfish.json.version>
-        <wildfly.common.version>1.2.0.Final</wildfly.common.version>
+        <wildfly.common.version>1.4.0.Final</wildfly.common.version>
 
         <!-- Authorization Drools Policy Provider -->
         <version.org.drools>6.5.0.Final</version.org.drools>
@@ -261,8 +263,8 @@
             </dependency>
             <dependency>
                 <groupId>org.jboss.spec.javax.ws.rs</groupId>
-                <artifactId>jboss-jaxrs-api_2.0_spec</artifactId>
+                <artifactId>jboss-jaxrs-api_2.1_spec</artifactId>
-                <version>${jboss-jaxrs-api_2.0_spec}</version>
+                <version>${jboss-jaxrs-api_2.1_spec}</version>
             </dependency>
             <dependency>
                 <groupId>org.jboss.resteasy</groupId>
@@ -711,8 +713,18 @@
             <dependency>
                 <groupId>org.jboss.aesh</groupId>
                 <artifactId>aesh</artifactId>
+                <version>${jboss.aesh.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.aesh</groupId>
+                <artifactId>aesh</artifactId>
                 <version>${aesh.version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.aesh</groupId>
+                <artifactId>aesh-readline</artifactId>
+                <version>${aesh.readline.version}</version>
+            </dependency>
 
             <!-- keycloak -->
             <dependency>

services/pom.xml

@@ -126,7 +126,7 @@
         </dependency>
         <dependency>
             <groupId>org.jboss.spec.javax.ws.rs</groupId>
-            <artifactId>jboss-jaxrs-api_2.0_spec</artifactId>
+            <artifactId>jboss-jaxrs-api_2.1_spec</artifactId>
         </dependency>
         <dependency>
             <groupId>org.jboss.spec.javax.transaction</groupId>

services/src/main/java/org/keycloak/services/managers/AuthSessionId.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.services.managers;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+class AuthSessionId {
+
+    // Decoded ID of authenticationSession WITHOUT route attached (EG. "5e161e00-d426-4ea6-98e9-52eb9844e2d7")
+    private final String decodedId;
+
+    // Encoded ID of authenticationSession WITH route attached (EG. "5e161e00-d426-4ea6-98e9-52eb9844e2d7.node1")
+    private final String encodedId;
+
+    AuthSessionId(String decodedId, String encodedId) {
+        this.decodedId = decodedId;
+        this.encodedId = encodedId;
+    }
+
+
+    public String getDecodedId() {
+        return decodedId;
+    }
+
+    public String getEncodedId() {
+        return encodedId;
+    }
+}

services/src/main/java/org/keycloak/services/managers/AuthenticationSessionManager.java

@@ -71,17 +71,18 @@ public class AuthenticationSessionManager {
         return rootAuthSession;
     }
 
-    public RootAuthenticationSessionModel getCurrentRootAuthenticationSession(RealmModel realm) {
-        List<String> authSessionIds = getAuthSessionCookieIds(realm);
 
-        return authSessionIds.stream().map(id -> {
+    public RootAuthenticationSessionModel getCurrentRootAuthenticationSession(RealmModel realm) {
-            SimpleEntry<String, String> entry = decodeAuthSessionId(id);
+        List<String> authSessionCookies = getAuthSessionCookies(realm);
-            String sessionId = entry.getKey();
+
+        return authSessionCookies.stream().map(oldEncodedId -> {
+            AuthSessionId authSessionId = decodeAuthSessionId(oldEncodedId);
+            String sessionId = authSessionId.getDecodedId();
 
             RootAuthenticationSessionModel rootAuthSession = session.authenticationSessions().getRootAuthenticationSession(realm, sessionId);
 
             if (rootAuthSession != null) {
-                reencodeAuthSessionCookie(sessionId, entry.getValue(), realm);
+                reencodeAuthSessionCookie(oldEncodedId, authSessionId, realm);
                 return rootAuthSession;
             }
 
@@ -89,17 +90,18 @@ public class AuthenticationSessionManager {
         }).filter(authSession -> Objects.nonNull(authSession)).findFirst().orElse(null);
     }
 
-    public UserSessionModel getUserSessionFromAuthCookie(RealmModel realm) {
-        List<String> authSessionIds = getAuthSessionCookieIds(realm);
 
-        return authSessionIds.stream().map(id -> {
+    public UserSessionModel getUserSessionFromAuthCookie(RealmModel realm) {
-            SimpleEntry<String, String> entry = decodeAuthSessionId(id);
+        List<String> authSessionCookies = getAuthSessionCookies(realm);
-            String sessionId = entry.getKey();
+
+        return authSessionCookies.stream().map(oldEncodedId -> {
+            AuthSessionId authSessionId = decodeAuthSessionId(oldEncodedId);
+            String sessionId = authSessionId.getDecodedId();
 
             UserSessionModel userSession = session.sessions().getUserSession(realm, sessionId);
 
             if (userSession != null) {
-                reencodeAuthSessionCookie(sessionId, entry.getValue(), realm);
+                reencodeAuthSessionCookie(oldEncodedId, authSessionId, realm);
                 return userSession;
             }
 
@@ -114,16 +116,16 @@ public class AuthenticationSessionManager {
      * @return
      */
     public AuthenticationSessionModel getCurrentAuthenticationSession(RealmModel realm, ClientModel client, String tabId) {
-        List<String> authSessionIds = getAuthSessionCookieIds(realm);
+        List<String> authSessionCookies = getAuthSessionCookies(realm);
 
-        return authSessionIds.stream().map(id -> {
+        return authSessionCookies.stream().map(oldEncodedId -> {
-            SimpleEntry<String, String> entry = decodeAuthSessionId(id);
+            AuthSessionId authSessionId = decodeAuthSessionId(oldEncodedId);
-            String sessionId = entry.getKey();
+            String sessionId = authSessionId.getDecodedId();
 
             AuthenticationSessionModel authSession = getAuthenticationSessionByIdAndClient(realm, sessionId, client, tabId);
 
             if (authSession != null) {
-                reencodeAuthSessionCookie(sessionId, entry.getValue(), realm);
+                reencodeAuthSessionCookie(oldEncodedId, authSessionId, realm);
                 return authSession;
             }
 
@@ -132,6 +134,10 @@ public class AuthenticationSessionManager {
     }
 
 
+    /**
+     * @param authSessionId decoded authSessionId (without route info attached)
+     * @param realm
+     */
     public void setAuthSessionCookie(String authSessionId, RealmModel realm) {
         UriInfo uriInfo = session.getContext().getUri();
         String cookiePath = AuthenticationManager.getRealmCookiePath(realm, uriInfo);
@@ -146,23 +152,36 @@ public class AuthenticationSessionManager {
         log.debugf("Set AUTH_SESSION_ID cookie with value %s", encodedAuthSessionId);
     }
 
-    public SimpleEntry<String, String> decodeAuthSessionId(String authSessionId) {
+
-        log.debugf("Found AUTH_SESSION_ID cookie with value %s", authSessionId);
+    /**
+     *
+     * @param encodedAuthSessionId encoded ID with attached route in cluster environment (EG. "5e161e00-d426-4ea6-98e9-52eb9844e2d7.node1" )
+     * @return object with decoded and actually encoded authSessionId
+     */
+    AuthSessionId decodeAuthSessionId(String encodedAuthSessionId) {
+        log.debugf("Found AUTH_SESSION_ID cookie with value %s", encodedAuthSessionId);
         StickySessionEncoderProvider encoder = session.getProvider(StickySessionEncoderProvider.class);
-        String decodedAuthSessionId = encoder.decodeSessionId(authSessionId);
+        String decodedAuthSessionId = encoder.decodeSessionId(encodedAuthSessionId);
         String reencoded = encoder.encodeSessionId(decodedAuthSessionId);
 
-        return new SimpleEntry(decodedAuthSessionId, reencoded);
+        return new AuthSessionId(decodedAuthSessionId, reencoded);
     }
 
-    public void reencodeAuthSessionCookie(String decodedAuthSessionId, String reencodedAuthSessionId, RealmModel realm) {
+
-        if (!decodedAuthSessionId.equals(reencodedAuthSessionId)) {
+    void reencodeAuthSessionCookie(String oldEncodedAuthSessionId, AuthSessionId newAuthSessionId, RealmModel realm) {
-            log.debugf("Route changed. Will update authentication session cookie");
+        if (!oldEncodedAuthSessionId.equals(newAuthSessionId.getEncodedId())) {
-            setAuthSessionCookie(decodedAuthSessionId, realm);
+            log.debugf("Route changed. Will update authentication session cookie. Old: '%s', New: '%s'", oldEncodedAuthSessionId,
+                    newAuthSessionId.getEncodedId());
+            setAuthSessionCookie(newAuthSessionId.getDecodedId(), realm);
         }
     }
 
-    public List<String> getAuthSessionCookieIds(RealmModel realm) {
+
+    /**
+     * @param realm
+     * @return list of the values of AUTH_SESSION_ID cookies. It is assumed that values could be encoded with route added (EG. "5e161e00-d426-4ea6-98e9-52eb9844e2d7.node1" )
+     */
+    List<String> getAuthSessionCookies(RealmModel realm) {
         Set<String> cookiesVal = CookieHelper.getCookieValue(AUTH_SESSION_ID);
 
         if (cookiesVal.size() > 1) {

services/src/main/java/org/keycloak/services/managers/UserSessionCrossDCManager.java

@@ -62,11 +62,11 @@ public class UserSessionCrossDCManager {
 
     // Just check if userSession also exists on remoteCache. It can happen that logout happened on 2nd DC and userSession is already removed on remoteCache and this DC wasn't yet notified
     public UserSessionModel getUserSessionIfExistsRemotely(AuthenticationSessionManager asm, RealmModel realm) {
-        List<String> sessionIds = asm.getAuthSessionCookieIds(realm);
+        List<String> sessionCookies = asm.getAuthSessionCookies(realm);
 
-        return sessionIds.stream().map(id -> {
+        return sessionCookies.stream().map(oldEncodedId -> {
-            SimpleEntry<String, String> entry = asm.decodeAuthSessionId(id);
+            AuthSessionId authSessionId = asm.decodeAuthSessionId(oldEncodedId);
-            String sessionId = entry.getKey();
+            String sessionId = authSessionId.getDecodedId();
 
             // This will remove userSession "locally" if it doesn't exists on remoteCache
             kcSession.sessions().getUserSessionWithPredicate(realm, sessionId, false, (UserSessionModel userSession2) -> userSession2 == null);
@@ -74,7 +74,7 @@ public class UserSessionCrossDCManager {
             UserSessionModel userSession = kcSession.sessions().getUserSession(realm, sessionId);
 
             if (userSession != null) {
-                asm.reencodeAuthSessionCookie(sessionId, entry.getValue(), realm);
+                asm.reencodeAuthSessionCookie(oldEncodedId, authSessionId, realm);
                 return userSession;
             }
 

testsuite/integration-arquillian/HOW-TO-RUN.md

@@ -287,6 +287,17 @@ This will start latest Keycloak and import the realm JSON file, which was previo
       -Dmigrated.auth.server.version=1.9.8.Final
 
 
+## Server configuration migration test
+This will compare if Wildfly configuration files (standalone.xml, standalone-ha.xml, domain.xml) 
+are correctly migrated from previous version
+
+    mvn -f testsuite/integration-arquillian/tests/other/server-config-migration/pom.xml \
+      clean install \
+      -Dmigrated.version=1.9.8.Final-redhat-1
+      
+For the available versions, take a look at the directory [tests/other/server-config-migration/src/test/resources/standalone](tests/other/server-config-migration/src/test/resources/standalone) 
+
+      
 ## Admin Console UI tests
 The UI tests are real-life, UI focused integration tests. Hence they do not support the default HtmlUnit browser. Only the following real-life browsers are supported: Mozilla Firefox, Google Chrome and Internet Explorer. For details on how to run the tests with these browsers, please refer to [Different Browsers](#different-browsers) chapter.
 

testsuite/integration-arquillian/servers/app-server/tomcat/pom.xml

@@ -112,7 +112,7 @@
                                         </artifactItem>
                                         <artifactItem>
                                             <groupId>org.jboss.spec.javax.ws.rs</groupId>
-                                            <artifactId>jboss-jaxrs-api_2.0_spec</artifactId>
+                                            <artifactId>jboss-jaxrs-api_2.1_spec</artifactId>
                                         </artifactItem>                                        
                                         <artifactItem>
                                             <groupId>org.jboss.resteasy</groupId>

testsuite/integration-arquillian/servers/auth-server/jboss/common/ispn-cache-owners.xsl

@@ -1,6 +1,6 @@
 <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
                 xmlns:xalan="http://xml.apache.org/xalan"
-                xmlns:i="urn:jboss:domain:infinispan:4.0"
+                xmlns:i="urn:jboss:domain:infinispan:6.0"
                 version="2.0"
                 exclude-result-prefixes="xalan i">
 
@@ -23,11 +23,21 @@
             <xsl:value-of select="$sessionCacheOwners"/>
         </xsl:attribute>
     </xsl:template>
+    <xsl:template match="//i:cache-container/i:distributed-cache[@name='clientSessions']/@owners">
+        <xsl:attribute name="owners">
+            <xsl:value-of select="$sessionCacheOwners"/>
+        </xsl:attribute>
+    </xsl:template>
     <xsl:template match="//i:cache-container/i:distributed-cache[@name='offlineSessions']/@owners">
         <xsl:attribute name="owners">
             <xsl:value-of select="$offlineSessionCacheOwners"/>
         </xsl:attribute>
     </xsl:template>
+    <xsl:template match="//i:cache-container/i:distributed-cache[@name='offlineClientSessions']/@owners">
+        <xsl:attribute name="owners">
+            <xsl:value-of select="$offlineSessionCacheOwners"/>
+        </xsl:attribute>
+    </xsl:template>
     <xsl:template match="//i:cache-container/i:distributed-cache[@name='loginFailures']/@owners">
         <xsl:attribute name="owners">
             <xsl:value-of select="$loginFailureCacheOwners"/>

testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/resources/org/keycloak/testsuite/integration-arquillian-testsuite-providers/main/module.xml

@@ -35,6 +35,7 @@
         <module name="org.keycloak.keycloak-ldap-federation"/>
         <module name="org.infinispan"/>
         <module name="org.infinispan.client.hotrod"/>
+        <module name="org.jgroups"/>
         <module name="org.jboss.logging"/>
         <module name="org.jboss.resteasy.resteasy-jaxrs"/>
         <module name="javax.persistence.api"/>

testsuite/integration-arquillian/servers/cache-server/jboss/common/add-keycloak-caches.xsl

@@ -25,6 +25,7 @@
 
     <xsl:param name="local.site" />
     <xsl:param name="remote.site" />
+    <xsl:param name="transactions.enabled" />
 
     <xsl:variable name="nsCacheServer" select="'urn:infinispan:server:core:'"/>
     <xsl:variable name="nsJGroups" select="'urn:infinispan:server:jgroups:'"/>
@@ -36,7 +37,9 @@
             <xsl:apply-templates select="@* | node()" />
 
             <replicated-cache-configuration name="sessions-cfg" mode="SYNC" start="EAGER" batching="false">
-                <transaction mode="NON_DURABLE_XA" locking="PESSIMISTIC"/>
+                <xsl:if test="$transactions.enabled='true'">
+                    <transaction mode="NON_DURABLE_XA" locking="PESSIMISTIC"/>
+                </xsl:if>
                 <locking acquire-timeout="0" />
                 <backups>
                     <backup site="{$remote.site}" failure-policy="FAIL" strategy="SYNC" enabled="true">

testsuite/integration-arquillian/servers/cache-server/jboss/infinispan/pom.xml

@@ -35,6 +35,7 @@
         <cache.server.home>${containers.home}/${cache.server.container}</cache.server.home>
         
         <cache.server.jboss.cache-authorization-disabled>true</cache.server.jboss.cache-authorization-disabled>
+        <cache.server.jboss.jdg-transactions-enabled>false</cache.server.jboss.jdg-transactions-enabled>
         <cache.server.jboss.groupId>org.infinispan.server</cache.server.jboss.groupId>
         <cache.server.jboss.artifactId>infinispan-server</cache.server.jboss.artifactId>
         <cache.server.jboss.version>${infinispan.version}</cache.server.jboss.version>

testsuite/integration-arquillian/servers/cache-server/jboss/jdg/pom.xml

@@ -34,7 +34,8 @@
         <cache.server.container>cache-server-${cache.server}</cache.server.container>
         <cache.server.home>${containers.home}/${cache.server.container}</cache.server.home>
         
-        <cache.server.jboss.cache-authorization-disabled>false</cache.server.jboss.cache-authorization-disabled>
+        <cache.server.jboss.cache-authorization-disabled>true</cache.server.jboss.cache-authorization-disabled>
+        <cache.server.jboss.jdg-transactions-enabled>true</cache.server.jboss.jdg-transactions-enabled>
         <cache.server.jboss.groupId>org.infinispan.server</cache.server.jboss.groupId>
         <cache.server.jboss.artifactId>infinispan-server</cache.server.jboss.artifactId>
         <cache.server.jboss.version>${jdg.version}</cache.server.jboss.version>

testsuite/integration-arquillian/servers/cache-server/jboss/pom.xml

@@ -34,6 +34,7 @@
         <assembly.xml>${project.parent.basedir}/assembly.xml</assembly.xml>
         <cache.server.jboss.home>${containers.home}/${cache.server.jboss.unpacked.folder.name}</cache.server.jboss.home>
         <cache.server.jboss.cache-authorization-disabled>true</cache.server.jboss.cache-authorization-disabled>
+        <cache.server.jboss.jdg-transactions-enabled>true</cache.server.jboss.jdg-transactions-enabled>
         <security.xslt>security.xsl</security.xslt>
     </properties>
 
@@ -126,6 +127,10 @@
                                                     <name>remote.site</name>
                                                     <value>dc-1</value>
                                                 </parameter>
+                                                <parameter>
+                                                    <name>transactions.enabled</name>
+                                                    <value>${cache.server.jboss.jdg-transactions-enabled}</value>
+                                                </parameter>
                                             </parameters>
                                             <outputDir>${cache.server.jboss.home}/standalone/configuration</outputDir>
                                             <fileMappers>
@@ -152,6 +157,10 @@
                                                     <name>remote.site</name>
                                                     <value>dc-0</value>
                                                 </parameter>
+                                                <parameter>
+                                                    <name>transactions.enabled</name>
+                                                    <value>${cache.server.jboss.jdg-transactions-enabled}</value>
+                                                </parameter>
                                             </parameters>
                                             <outputDir>${cache.server.jboss.home}/standalone/configuration</outputDir>
                                             <fileMappers>

testsuite/integration-arquillian/servers/pom.xml

@@ -47,8 +47,8 @@
         <fuse62.version>6.2.1.redhat-084</fuse62.version>
         
         <!-- cache server versions -->
-        <!--<infinispan.version>9.0.1.Final</infinispan.version>--> <!-- Use same version like our infinispan version for now -->
+        <!--<infinispan.version>8.2.8.Final</infinispan.version>--><!-- Use same infinspan-server version as our version -->
-        <jdg.version>8.4.0.Final-redhat-2</jdg.version><!-- JDG 7.1.0 -->
+        <jdg.version>8.5.0.Final-redhat-9</jdg.version><!-- JDG 7.2.0 -->
         
         <jboss.default.worker.io-threads>16</jboss.default.worker.io-threads>
         <jboss.default.worker.task-max-threads>128</jboss.default.worker.task-max-threads>

testsuite/integration-arquillian/test-apps/photoz/photoz-restful-api/pom.xml

@@ -19,7 +19,7 @@
     <dependencies>
         <dependency>
             <groupId>org.jboss.spec.javax.ws.rs</groupId>
-            <artifactId>jboss-jaxrs-api_2.0_spec</artifactId>
+            <artifactId>jboss-jaxrs-api_2.1_spec</artifactId>
             <scope>provided</scope>
         </dependency>
         <dependency>

testsuite/integration-arquillian/test-apps/servlets/pom.xml

@@ -32,7 +32,7 @@
         </dependency>
         <dependency>
             <groupId>org.jboss.spec.javax.ws.rs</groupId>
-            <artifactId>jboss-jaxrs-api_2.0_spec</artifactId>
+            <artifactId>jboss-jaxrs-api_2.1_spec</artifactId>
         </dependency>
         <dependency>
             <groupId>org.keycloak</groupId>

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AuthenticationSessionClusterTest.java

@@ -146,7 +146,7 @@ public class AuthenticationSessionClusterTest extends AbstractClusterTest {
             // Check that route owner is always node1
             getTestingClientFor(backendNode(0)).server().run(session -> {
                 Cache authSessionCache = session.getProvider(InfinispanConnectionProvider.class).getCache(InfinispanConnectionProvider.AUTHENTICATION_SESSIONS_CACHE_NAME);
-                String keyOwner = InfinispanUtil.getKeyPrimaryOwnerAddress(authSessionCache, authSessionCookie);
+                String keyOwner = InfinispanUtil.getTopologyInfo(session).getRouteName(authSessionCache, authSessionCookie);
                 Assert.assertEquals("node1", keyOwner);
             });
         }

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/ActionTokenCrossDCTest.java

@@ -92,7 +92,7 @@ public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest {
         
         cacheDc0Node1Statistics.waitToBecomeAvailable(10, TimeUnit.SECONDS);
 
-        Comparable originalNumberOfEntries = cacheDc0Node0Statistics.getSingleStatistics(Constants.STAT_CACHE_NUMBER_OF_ENTRIES);
+        Comparable originalNumberOfEntries = cacheDc0Node0Statistics.getSingleStatistics(Constants.STAT_CACHE_NUMBER_OF_ENTRIES_IN_MEMORY);
 
         UserRepresentation userRep = new UserRepresentation();
         userRep.setEnabled(true);
@@ -112,7 +112,7 @@ public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest {
 
         String link = MailUtils.getPasswordResetEmailLink(message);
 
-        assertSingleStatistics(cacheDc0Node0Statistics, Constants.STAT_CACHE_NUMBER_OF_ENTRIES,
+        assertSingleStatistics(cacheDc0Node0Statistics, Constants.STAT_CACHE_NUMBER_OF_ENTRIES_IN_MEMORY,
           () -> driver.navigate().to(link),
           Matchers::is
         );
@@ -141,13 +141,15 @@ public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest {
         assertThat(PageUtils.getPageTitle(driver), containsString("Your account has been updated."));
 
         // Verify that there was an action token added in the node which was targetted by the link
-        assertThat(cacheDc0Node0Statistics.getSingleStatistics(Constants.STAT_CACHE_NUMBER_OF_ENTRIES), greaterThan(originalNumberOfEntries));
+        assertThat(cacheDc0Node0Statistics.getSingleStatistics(Constants.STAT_CACHE_NUMBER_OF_ENTRIES_IN_MEMORY), greaterThan(originalNumberOfEntries));
 
         disableDcOnLoadBalancer(DC.FIRST);
         enableDcOnLoadBalancer(DC.SECOND);
 
         // Make sure that after going to the link, the invalidated action token has been retrieved from Infinispan server cluster in the other DC
-        assertSingleStatistics(cacheDc1Node0Statistics, Constants.STAT_CACHE_NUMBER_OF_ENTRIES,
+        // NOTE: Using STAT_CACHE_NUMBER_OF_ENTRIES_IN_MEMORY as it doesn't contain the items from cacheLoader (remoteCache) until they are really loaded into the cache memory. That's the
+        // statistic, which is actually increased on dc1-node0 once the used actionToken is loaded to the cache (memory) from remoteCache
+        assertSingleStatistics(cacheDc1Node0Statistics, Constants.STAT_CACHE_NUMBER_OF_ENTRIES_IN_MEMORY,
           () -> driver.navigate().to(link),
           Matchers::greaterThan
         );

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/BruteForceCrossDCTest.java

@@ -168,6 +168,9 @@ public class BruteForceCrossDCTest extends AbstractAdminCrossDCTest {
         enableDcOnLoadBalancer(DC.FIRST);
         enableDcOnLoadBalancer(DC.SECOND);
 
+//        log.infof("Sleeping");
+//        Thread.sleep(3600000);
+
         // Clear all
         adminClient.realms().realm(REALM_NAME).attackDetection().clearAllBruteForce();
         assertStatistics("After brute force cleared", 0, 0, 0);
@@ -222,6 +225,8 @@ public class BruteForceCrossDCTest extends AbstractAdminCrossDCTest {
 
     @Test
     public void testBruteForceConcurrentUpdate() throws Exception {
+        //Thread.sleep(120000);
+
         // Enable 1st node on each DC only
         enableDcOnLoadBalancer(DC.FIRST);
         enableDcOnLoadBalancer(DC.SECOND);

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/ConcurrentLoginCrossDCTest.java

@@ -20,12 +20,10 @@ package org.keycloak.testsuite.crossdc;
 import java.util.ArrayList;
 import org.keycloak.admin.client.Keycloak;
 import org.keycloak.admin.client.resource.RealmResource;
-import java.util.List;
 
 import org.jboss.arquillian.container.test.api.ContainerController;
 import org.jboss.arquillian.test.api.ArquillianResource;
 import org.keycloak.testsuite.admin.concurrency.ConcurrentLoginTest;
-import org.keycloak.testsuite.arquillian.ContainerInfo;
 import org.keycloak.testsuite.arquillian.LoadBalancerController;
 import org.keycloak.testsuite.arquillian.annotation.LoadBalancer;
 import java.util.Arrays;

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/SessionExpirationCrossDCTest.java

@@ -192,7 +192,8 @@ public class SessionExpirationCrossDCTest extends AbstractAdminCrossDCTest {
             int clientSessions2 = getTestingClientForStartedNodeInDc(1).testing().cache(clientSessionsCacheName).size();
             int remoteSessions1 = (Integer) cacheDc1Statistics.getSingleStatistics(InfinispanStatistics.Constants.STAT_CACHE_NUMBER_OF_ENTRIES);
             int remoteSessions2 = (Integer) cacheDc2Statistics.getSingleStatistics(InfinispanStatistics.Constants.STAT_CACHE_NUMBER_OF_ENTRIES);
-            long messagesCount = (Long) channelStatisticsCrossDc.getSingleStatistics(InfinispanStatistics.Constants.STAT_CHANNEL_SENT_MESSAGES);
+            // Needs to use "received_messages" on Infinispan 9.2.4.Final.  Stats for "sent_messages" is always null
+            long messagesCount = (Long) channelStatisticsCrossDc.getSingleStatistics(InfinispanStatistics.Constants.STAT_CHANNEL_RECEIVED_MESSAGES);
             log.infof(messagePrefix + ": sessions1: %d, sessions2: %d, remoteSessions1: %d, remoteSessions2: %d, sentMessages: %d", sessions1, sessions2, remoteSessions1, remoteSessions2, messagesCount);
 
             Assert.assertEquals(sessions1, sessions1Expected);
@@ -432,6 +433,15 @@ public class SessionExpirationCrossDCTest extends AbstractAdminCrossDCTest {
         // Kill node2 now. Around 10 sessions (half of SESSIONS_COUNT) will be lost on Keycloak side. But not on infinispan side
         CrossDCTestEnricher.stopAuthServerBackendNode(DC.FIRST, 1);
 
+        // Assert it's still possible to refresh tokens. UserSessions, which were cleared from the Keycloak node, should be downloaded from remoteStore
+        int i1 = 0;
+        for (OAuthClient.AccessTokenResponse response : responses) {
+            i1++;
+            OAuthClient.AccessTokenResponse refreshTokenResponse = oauth.doRefreshTokenRequest(response.getRefreshToken(), "password");
+            Assert.assertNotNull("Failed in iteration " + i1, refreshTokenResponse.getRefreshToken());
+            Assert.assertNull("Failed in iteration " + i1, refreshTokenResponse.getError());
+        }
+
         channelStatisticsCrossDc.reset();
 
         // Increase offset a bit to ensure logout happens later then token issued time
@@ -662,7 +672,7 @@ public class SessionExpirationCrossDCTest extends AbstractAdminCrossDCTest {
         Retry.execute(() -> {
             int authSessions1 = getTestingClientForStartedNodeInDc(0).testing().cache(InfinispanConnectionProvider.AUTHENTICATION_SESSIONS_CACHE_NAME).size();
             int authSessions2 = getTestingClientForStartedNodeInDc(1).testing().cache(InfinispanConnectionProvider.AUTHENTICATION_SESSIONS_CACHE_NAME).size();
-            long messagesCount = (Long) channelStatisticsCrossDc.getSingleStatistics(InfinispanStatistics.Constants.STAT_CHANNEL_SENT_MESSAGES);
+            long messagesCount = (Long) channelStatisticsCrossDc.getSingleStatistics(InfinispanStatistics.Constants.STAT_CHANNEL_RECEIVED_MESSAGES);
             log.infof(messagePrefix + ": authSessions1: %d, authSessions2: %d, sentMessages: %d", authSessions1, authSessions2, messagesCount);
 
             int diff1 = authSessions1 - authSessions01;

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/error/UncaughtErrorPageTest.java

@@ -1,16 +1,22 @@
 package org.keycloak.testsuite.error;
 
 import org.jboss.arquillian.graphene.page.Page;
+import org.junit.Assert;
 import org.junit.Test;
 import org.keycloak.admin.client.resource.RealmResource;
+import org.keycloak.common.util.StreamUtil;
 import org.keycloak.representations.idm.RealmRepresentation;
 import org.keycloak.testsuite.AbstractKeycloakTest;
 import org.keycloak.testsuite.pages.ErrorPage;
 
 import javax.ws.rs.core.Response;
+
+import java.io.IOException;
+import java.io.InputStream;
 import java.lang.reflect.Array;
 import java.net.MalformedURLException;
 import java.net.URI;
+import java.nio.charset.Charset;
 import java.util.Collections;
 import java.util.List;
 
@@ -43,10 +49,14 @@ public class UncaughtErrorPageTest extends AbstractKeycloakTest {
     }
 
     @Test
-    public void uncaughtErrorJson() {
+    public void uncaughtErrorJson() throws IOException {
         Response response = testingClient.testing().uncaughtError();
-        assertNull(response.getEntity());
         assertEquals(500, response.getStatus());
+
+        InputStream is = (InputStream) response.getEntity();
+        String responseString = StreamUtil.readString(is, Charset.forName("UTF-8"));
+
+        Assert.assertTrue(responseString.contains("An internal server error has occurred"));
     }
 
     @Test

testsuite/integration-arquillian/tests/base/src/test/resources/META-INF/keycloak-server.json

@@ -126,7 +126,7 @@
             "sessionsOwners": "${keycloak.connectionsInfinispan.sessionsOwners:1}",
             "l1Lifespan": "${keycloak.connectionsInfinispan.l1Lifespan:600000}",
             "remoteStoreEnabled": "${keycloak.connectionsInfinispan.remoteStoreEnabled:false}",
-            "remoteStoreServer": "${keycloak.connectionsInfinispan.remoteStoreServer:localhost}",
+            "remoteStoreHost": "${keycloak.connectionsInfinispan.remoteStoreServer:localhost}",
             "remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort:11222}"
         }
     },

testsuite/integration-arquillian/tests/other/server-config-migration/README.md

@@ -27,7 +27,7 @@ Migration scripts are applied using **offline mode**. Temporary data are removed
 `maven-exec-plugin` is used to read migrated configs and saves the output to `${project.build.directory}/migrated-${config.name}.txt`
 
 ### `default-test`
-`org.keycloak.test.config.migrationConfigMigrationTest` is executed. It compares generated outputs from ${project.build.directory}
+`org.keycloak.test.config.migration.ConfigMigrationTest` is executed. It compares generated outputs from ${project.build.directory}
 
 If config outputs don't equal to each other, **by default** the test will compare outputs more deeply to get more readable output. It fails on first found difference.
 

testsuite/integration-arquillian/tests/other/server-config-migration/src/test/resources/domain/host-master-3.4.3.Final-redhat-2.xml

@@ -0,0 +1,185 @@
+<?xml version='1.0' encoding='UTF-8'?>
+
+<host xmlns="urn:jboss:domain:5.0" name="master">
+    <extensions>
+        <extension module="org.jboss.as.jmx"/>
+        <extension module="org.wildfly.extension.core-management"/>
+        <extension module="org.wildfly.extension.elytron"/>
+    </extensions>
+    <management>
+        <security-realms>
+            <security-realm name="ManagementRealm">
+                <authentication>
+                    <local default-user="$local" skip-group-loading="true"/>
+                    <properties path="mgmt-users.properties" relative-to="jboss.domain.config.dir"/>
+                </authentication>
+                <authorization map-groups-to-roles="false">
+                    <properties path="mgmt-groups.properties" relative-to="jboss.domain.config.dir"/>
+                </authorization>
+            </security-realm>
+            <security-realm name="ApplicationRealm">
+                <server-identities>
+                    <ssl>
+                        <keystore path="application.keystore" relative-to="jboss.domain.config.dir" keystore-password="password" alias="server" key-password="password" generate-self-signed-certificate-host="localhost"/>
+                    </ssl>
+                </server-identities>
+                <authentication>
+                    <local default-user="$local" allowed-users="*" skip-group-loading="true"/>
+                    <properties path="application-users.properties" relative-to="jboss.domain.config.dir"/>
+                </authentication>
+                <authorization>
+                    <properties path="application-roles.properties" relative-to="jboss.domain.config.dir"/>
+                </authorization>
+            </security-realm>
+        </security-realms>
+        <audit-log>
+            <formatters>
+                <json-formatter name="json-formatter"/>
+            </formatters>
+            <handlers>
+                <file-handler name="host-file" formatter="json-formatter" path="audit-log.log" relative-to="jboss.domain.data.dir"/>
+                <file-handler name="server-file" formatter="json-formatter" path="audit-log.log" relative-to="jboss.server.data.dir"/>
+            </handlers>
+            <logger log-boot="true" log-read-only="false" enabled="false">
+                <handlers>
+                    <handler name="host-file"/>
+                </handlers>
+            </logger>
+            <server-logger log-boot="true" log-read-only="false" enabled="false">
+                <handlers>
+                    <handler name="server-file"/>
+                </handlers>
+            </server-logger>
+        </audit-log>
+        <management-interfaces>
+            <native-interface security-realm="ManagementRealm">
+                <socket interface="management" port="${jboss.management.native.port:9999}"/>
+            </native-interface>
+            <http-interface security-realm="ManagementRealm">
+                <http-upgrade enabled="true"/>
+                <socket interface="management" port="${jboss.management.http.port:9990}"/>
+            </http-interface>
+        </management-interfaces>
+    </management>
+    <domain-controller>
+        <local/>
+    </domain-controller>
+    <interfaces>
+        <interface name="management">
+            <inet-address value="${jboss.bind.address.management:127.0.0.1}"/>
+        </interface>
+        <interface name="public">
+            <inet-address value="${jboss.bind.address:127.0.0.1}"/>
+        </interface>
+    </interfaces>
+    <jvms>
+        <jvm name="default">
+            <heap size="64m" max-size="256m"/>
+            <jvm-options>
+                <option value="-server"/>
+                <option value="-XX:MetaspaceSize=96m"/>
+                <option value="-XX:MaxMetaspaceSize=256m"/>
+            </jvm-options>
+        </jvm>
+    </jvms>
+    <servers>
+        <!-- load-balancer should be removed in production systems and replaced with a better softare or hardare based one -->
+        <server name="load-balancer" group="load-balancer-group"/>
+        <server name="server-one" group="auth-server-group" auto-start="true">
+            <!--
+              ~  Remote JPDA debugging for a specific server
+              ~             <jvm name="default">
+              ~               <jvm-options>
+              ~                 <option value="-agentlib:jdwp=transport=dt_socket,address=8787,server=y,suspend=n"/>
+              ~               </jvm-options>
+              ~            </jvm>
+              ~            
+              -->
+            <!--
+              ~  server-two avoids port conflicts by incrementing the ports in
+              ~                  the default socket-group declared in the server-group 
+              -->
+            <socket-bindings port-offset="150"/>
+        </server>
+    </servers>
+    <profile>
+        <subsystem xmlns="urn:jboss:domain:core-management:1.0"/>
+        <subsystem xmlns="urn:jboss:domain:jmx:1.3">
+            <expose-resolved-model/>
+            <expose-expression-model/>
+            <remoting-connector/>
+        </subsystem>
+        <subsystem xmlns="urn:wildfly:elytron:1.2" final-providers="combined-providers" disallowed-providers="OracleUcrypto">
+            <providers>
+                <aggregate-providers name="combined-providers">
+                    <providers name="elytron"/>
+                    <providers name="openssl"/>
+                </aggregate-providers>
+                <provider-loader name="elytron" module="org.wildfly.security.elytron"/>
+                <provider-loader name="openssl" module="org.wildfly.openssl"/>
+            </providers>
+            <audit-logging>
+                <file-audit-log name="local-audit" path="audit.log" relative-to="jboss.domain.log.dir" format="JSON"/>
+            </audit-logging>
+            <security-domains>
+                <security-domain name="ManagementDomain" default-realm="ManagementRealm" permission-mapper="default-permission-mapper">
+                    <realm name="ManagementRealm" role-decoder="groups-to-roles"/>
+                    <realm name="local" role-mapper="super-user-mapper"/>
+                </security-domain>
+            </security-domains>
+            <security-realms>
+                <identity-realm name="local" identity="$local"/>
+                <properties-realm name="ManagementRealm">
+                    <users-properties path="mgmt-users.properties" relative-to="jboss.domain.config.dir" digest-realm-name="ManagementRealm"/>
+                    <groups-properties path="mgmt-groups.properties" relative-to="jboss.domain.config.dir"/>
+                </properties-realm>
+            </security-realms>
+            <mappers>
+                <simple-permission-mapper name="default-permission-mapper" mapping-mode="first">
+                    <permission-mapping>
+                        <principal name="anonymous"/>
+                    </permission-mapping>
+                    <permission-mapping match-all="true">
+                        <permission class-name="org.wildfly.security.auth.permission.LoginPermission"/>
+                    </permission-mapping>
+                </simple-permission-mapper>
+                <constant-realm-mapper name="local" realm-name="local"/>
+                <simple-role-decoder name="groups-to-roles" attribute="groups"/>
+                <constant-role-mapper name="super-user-mapper">
+                    <role name="SuperUser"/>
+                </constant-role-mapper>
+            </mappers>
+            <http>
+                <http-authentication-factory name="management-http-authentication" http-server-mechanism-factory="global" security-domain="ManagementDomain">
+                    <mechanism-configuration>
+                        <mechanism mechanism-name="BASIC">
+                            <mechanism-realm realm-name="Management Realm"/>
+                        </mechanism>
+                    </mechanism-configuration>
+                </http-authentication-factory>
+                <provider-http-server-mechanism-factory name="global"/>
+            </http>
+            <sasl>
+                <sasl-authentication-factory name="management-sasl-authentication" sasl-server-factory="configured" security-domain="ManagementDomain">
+                    <mechanism-configuration>
+                        <mechanism mechanism-name="JBOSS-LOCAL-USER" realm-mapper="local"/>
+                        <mechanism mechanism-name="DIGEST-MD5">
+                            <mechanism-realm realm-name="ManagementRealm"/>
+                        </mechanism>
+                    </mechanism-configuration>
+                </sasl-authentication-factory>
+                <configurable-sasl-server-factory name="configured" sasl-server-factory="elytron">
+                    <properties>
+                        <property name="wildfly.sasl.local-user.default-user" value="$local"/>
+                    </properties>
+                </configurable-sasl-server-factory>
+                <mechanism-provider-filtering-sasl-server-factory name="elytron" sasl-server-factory="global">
+                    <filters>
+                        <filter provider-name="WildFlyElytron"/>
+                    </filters>
+                </mechanism-provider-filtering-sasl-server-factory>
+                <provider-sasl-server-factory name="global"/>
+            </sasl>
+        </subsystem>
+    </profile>
+</host>

testsuite/integration-arquillian/tests/other/server-config-migration/src/test/resources/standalone/standalone-3.4.3.Final-redhat-2.xml

@@ -0,0 +1,573 @@
+<?xml version='1.0' encoding='UTF-8'?>
+
+<server xmlns="urn:jboss:domain:5.0">
+    <extensions>
+        <extension module="org.jboss.as.clustering.infinispan"/>
+        <extension module="org.jboss.as.connector"/>
+        <extension module="org.jboss.as.deployment-scanner"/>
+        <extension module="org.jboss.as.ee"/>
+        <extension module="org.jboss.as.ejb3"/>
+        <extension module="org.jboss.as.jaxrs"/>
+        <extension module="org.jboss.as.jmx"/>
+        <extension module="org.jboss.as.jpa"/>
+        <extension module="org.jboss.as.logging"/>
+        <extension module="org.jboss.as.mail"/>
+        <extension module="org.jboss.as.naming"/>
+        <extension module="org.jboss.as.remoting"/>
+        <extension module="org.jboss.as.security"/>
+        <extension module="org.jboss.as.transactions"/>
+        <extension module="org.keycloak.keycloak-server-subsystem"/>
+        <extension module="org.wildfly.extension.bean-validation"/>
+        <extension module="org.wildfly.extension.elytron"/>
+        <extension module="org.wildfly.extension.io"/>
+        <extension module="org.wildfly.extension.request-controller"/>
+        <extension module="org.wildfly.extension.security.manager"/>
+        <extension module="org.wildfly.extension.undertow"/>
+    </extensions>
+    <management>
+        <security-realms>
+            <security-realm name="ManagementRealm">
+                <authentication>
+                    <local default-user="$local" skip-group-loading="true"/>
+                    <properties path="mgmt-users.properties" relative-to="jboss.server.config.dir"/>
+                </authentication>
+                <authorization map-groups-to-roles="false">
+                    <properties path="mgmt-groups.properties" relative-to="jboss.server.config.dir"/>
+                </authorization>
+            </security-realm>
+            <security-realm name="ApplicationRealm">
+                <server-identities>
+                    <ssl>
+                        <keystore path="application.keystore" relative-to="jboss.server.config.dir" keystore-password="password" alias="server" key-password="password" generate-self-signed-certificate-host="localhost"/>
+                    </ssl>
+                </server-identities>
+                <authentication>
+                    <local default-user="$local" allowed-users="*" skip-group-loading="true"/>
+                    <properties path="application-users.properties" relative-to="jboss.server.config.dir"/>
+                </authentication>
+                <authorization>
+                    <properties path="application-roles.properties" relative-to="jboss.server.config.dir"/>
+                </authorization>
+            </security-realm>
+        </security-realms>
+        <audit-log>
+            <formatters>
+                <json-formatter name="json-formatter"/>
+            </formatters>
+            <handlers>
+                <file-handler name="file" formatter="json-formatter" path="audit-log.log" relative-to="jboss.server.data.dir"/>
+            </handlers>
+            <logger log-boot="true" log-read-only="false" enabled="false">
+                <handlers>
+                    <handler name="file"/>
+                </handlers>
+            </logger>
+        </audit-log>
+        <management-interfaces>
+            <http-interface security-realm="ManagementRealm">
+                <http-upgrade enabled="true"/>
+                <socket-binding http="management-http"/>
+            </http-interface>
+        </management-interfaces>
+        <access-control provider="simple">
+            <role-mapping>
+                <role name="SuperUser">
+                    <include>
+                        <user name="$local"/>
+                    </include>
+                </role>
+            </role-mapping>
+        </access-control>
+    </management>
+    <profile>
+        <subsystem xmlns="urn:jboss:domain:logging:3.0">
+            <console-handler name="CONSOLE">
+                <level name="INFO"/>
+                <formatter>
+                    <named-formatter name="COLOR-PATTERN"/>
+                </formatter>
+            </console-handler>
+            <periodic-rotating-file-handler name="FILE" autoflush="true">
+                <formatter>
+                    <named-formatter name="PATTERN"/>
+                </formatter>
+                <file relative-to="jboss.server.log.dir" path="server.log"/>
+                <suffix value=".yyyy-MM-dd"/>
+                <append value="true"/>
+            </periodic-rotating-file-handler>
+            <logger category="com.arjuna">
+                <level name="WARN"/>
+            </logger>
+            <logger category="org.jboss.as.config">
+                <level name="DEBUG"/>
+            </logger>
+            <logger category="sun.rmi">
+                <level name="WARN"/>
+            </logger>
+            <root-logger>
+                <level name="INFO"/>
+                <handlers>
+                    <handler name="CONSOLE"/>
+                    <handler name="FILE"/>
+                </handlers>
+            </root-logger>
+            <formatter name="PATTERN">
+                <pattern-formatter pattern="%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%c] (%t) %s%e%n"/>
+            </formatter>
+            <formatter name="COLOR-PATTERN">
+                <pattern-formatter pattern="%K{level}%d{HH:mm:ss,SSS} %-5p [%c] (%t) %s%e%n"/>
+            </formatter>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:bean-validation:1.0"/>
+        <subsystem xmlns="urn:jboss:domain:datasources:5.0">
+            <datasources>
+                <datasource jndi-name="java:jboss/datasources/ExampleDS" pool-name="ExampleDS" enabled="true" use-java-context="true">
+                    <connection-url>jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE</connection-url>
+                    <driver>h2</driver>
+                    <security>
+                        <user-name>sa</user-name>
+                        <password>sa</password>
+                    </security>
+                </datasource>
+                <datasource jndi-name="java:jboss/datasources/KeycloakDS" pool-name="KeycloakDS" enabled="true" use-java-context="true">
+                    <connection-url>jdbc:h2:${jboss.server.data.dir}/keycloak;AUTO_SERVER=TRUE</connection-url>
+                    <driver>h2</driver>
+                    <security>
+                        <user-name>sa</user-name>
+                        <password>sa</password>
+                    </security>
+                </datasource>
+                <drivers>
+                    <driver name="h2" module="com.h2database.h2">
+                        <xa-datasource-class>org.h2.jdbcx.JdbcDataSource</xa-datasource-class>
+                    </driver>
+                </drivers>
+            </datasources>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:deployment-scanner:2.0">
+            <deployment-scanner path="deployments" relative-to="jboss.server.base.dir" scan-interval="5000" runtime-failure-causes-rollback="${jboss.deployment.scanner.rollback.on.failure:false}"/>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:ee:4.0">
+            <spec-descriptor-property-replacement>false</spec-descriptor-property-replacement>
+            <concurrent>
+                <context-services>
+                    <context-service name="default" jndi-name="java:jboss/ee/concurrency/context/default" use-transaction-setup-provider="true"/>
+                </context-services>
+                <managed-thread-factories>
+                    <managed-thread-factory name="default" jndi-name="java:jboss/ee/concurrency/factory/default" context-service="default"/>
+                </managed-thread-factories>
+                <managed-executor-services>
+                    <managed-executor-service name="default" jndi-name="java:jboss/ee/concurrency/executor/default" context-service="default" hung-task-threshold="60000" keepalive-time="5000"/>
+                </managed-executor-services>
+                <managed-scheduled-executor-services>
+                    <managed-scheduled-executor-service name="default" jndi-name="java:jboss/ee/concurrency/scheduler/default" context-service="default" hung-task-threshold="60000" keepalive-time="3000"/>
+                </managed-scheduled-executor-services>
+            </concurrent>
+            <default-bindings context-service="java:jboss/ee/concurrency/context/default" datasource="java:jboss/datasources/ExampleDS" managed-executor-service="java:jboss/ee/concurrency/executor/default" managed-scheduled-executor-service="java:jboss/ee/concurrency/scheduler/default" managed-thread-factory="java:jboss/ee/concurrency/factory/default"/>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:ejb3:5.0">
+            <session-bean>
+                <stateless>
+                    <bean-instance-pool-ref pool-name="slsb-strict-max-pool"/>
+                </stateless>
+                <stateful default-access-timeout="5000" cache-ref="simple" passivation-disabled-cache-ref="simple"/>
+                <singleton default-access-timeout="5000"/>
+            </session-bean>
+            <pools>
+                <bean-instance-pools>
+                    <strict-max-pool name="slsb-strict-max-pool" derive-size="from-worker-pools" instance-acquisition-timeout="5" instance-acquisition-timeout-unit="MINUTES"/>
+                    <strict-max-pool name="mdb-strict-max-pool" derive-size="from-cpu-count" instance-acquisition-timeout="5" instance-acquisition-timeout-unit="MINUTES"/>
+                </bean-instance-pools>
+            </pools>
+            <caches>
+                <cache name="simple"/>
+                <cache name="distributable" passivation-store-ref="infinispan" aliases="passivating clustered"/>
+            </caches>
+            <passivation-stores>
+                <passivation-store name="infinispan" cache-container="ejb" max-size="10000"/>
+            </passivation-stores>
+            <async thread-pool-name="default"/>
+            <timer-service thread-pool-name="default" default-data-store="default-file-store">
+                <data-stores>
+                    <file-data-store name="default-file-store" path="timer-service-data" relative-to="jboss.server.data.dir"/>
+                </data-stores>
+            </timer-service>
+            <remote connector-ref="http-remoting-connector" thread-pool-name="default">
+                <channel-creation-options>
+                    <option name="READ_TIMEOUT" value="${prop.remoting-connector.read.timeout:20}" type="xnio"/>
+                    <option name="MAX_OUTBOUND_MESSAGES" value="1234" type="remoting"/>
+                </channel-creation-options>
+            </remote>
+            <thread-pools>
+                <thread-pool name="default">
+                    <max-threads count="10"/>
+                    <keepalive-time time="100" unit="milliseconds"/>
+                </thread-pool>
+            </thread-pools>
+            <default-security-domain value="other"/>
+            <default-missing-method-permissions-deny-access value="true"/>
+            <log-system-exceptions value="true"/>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:io:2.0">
+            <worker name="default"/>
+            <buffer-pool name="default"/>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:infinispan:4.0">
+            <cache-container name="keycloak" jndi-name="infinispan/Keycloak">
+                <local-cache name="realms">
+                    <eviction max-entries="10000" strategy="LRU"/>
+                </local-cache>
+                <local-cache name="users">
+                    <eviction max-entries="10000" strategy="LRU"/>
+                </local-cache>
+                <local-cache name="sessions"/>
+                <local-cache name="authenticationSessions"/>
+                <local-cache name="offlineSessions"/>
+                <local-cache name="clientSessions"/>
+                <local-cache name="offlineClientSessions"/>
+                <local-cache name="loginFailures"/>
+                <local-cache name="work"/>
+                <local-cache name="authorization">
+                    <eviction max-entries="10000" strategy="LRU"/>
+                </local-cache>
+                <local-cache name="keys">
+                    <eviction max-entries="1000" strategy="LRU"/>
+                    <expiration max-idle="3600000"/>
+                </local-cache>
+                <local-cache name="actionTokens">
+                    <eviction max-entries="-1" strategy="NONE"/>
+                    <expiration max-idle="-1" interval="300000"/>
+                </local-cache>
+            </cache-container>
+            <cache-container name="server" default-cache="default" module="org.wildfly.clustering.server">
+                <local-cache name="default">
+                    <transaction mode="BATCH"/>
+                </local-cache>
+            </cache-container>
+            <cache-container name="web" default-cache="passivation" module="org.wildfly.clustering.web.infinispan">
+                <local-cache name="passivation">
+                    <locking isolation="REPEATABLE_READ"/>
+                    <transaction mode="BATCH"/>
+                    <file-store passivation="true" purge="false"/>
+                </local-cache>
+            </cache-container>
+            <cache-container name="ejb" aliases="sfsb" default-cache="passivation" module="org.wildfly.clustering.ejb.infinispan">
+                <local-cache name="passivation">
+                    <locking isolation="REPEATABLE_READ"/>
+                    <transaction mode="BATCH"/>
+                    <file-store passivation="true" purge="false"/>
+                </local-cache>
+            </cache-container>
+            <cache-container name="hibernate" module="org.hibernate.infinispan">
+                <local-cache name="entity">
+                    <transaction mode="NON_XA"/>
+                    <eviction strategy="LRU" max-entries="10000"/>
+                    <expiration max-idle="100000"/>
+                </local-cache>
+                <local-cache name="local-query">
+                    <eviction strategy="LRU" max-entries="10000"/>
+                    <expiration max-idle="100000"/>
+                </local-cache>
+                <local-cache name="timestamps"/>
+            </cache-container>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:jaxrs:1.0"/>
+        <subsystem xmlns="urn:jboss:domain:jca:5.0">
+            <archive-validation enabled="true" fail-on-error="true" fail-on-warn="false"/>
+            <bean-validation enabled="true"/>
+            <default-workmanager>
+                <short-running-threads>
+                    <core-threads count="50"/>
+                    <queue-length count="50"/>
+                    <max-threads count="50"/>
+                    <keepalive-time time="10" unit="seconds"/>
+                </short-running-threads>
+                <long-running-threads>
+                    <core-threads count="50"/>
+                    <queue-length count="50"/>
+                    <max-threads count="50"/>
+                    <keepalive-time time="10" unit="seconds"/>
+                </long-running-threads>
+            </default-workmanager>
+            <cached-connection-manager/>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:jmx:1.3">
+            <expose-resolved-model/>
+            <expose-expression-model/>
+            <remoting-connector/>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:jpa:1.1">
+            <jpa default-datasource="" default-extended-persistence-inheritance="DEEP"/>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:mail:3.0">
+            <mail-session name="default" jndi-name="java:jboss/mail/Default">
+                <smtp-server outbound-socket-binding-ref="mail-smtp"/>
+            </mail-session>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:naming:2.0">
+            <remote-naming/>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:remoting:4.0">
+            <endpoint/>
+            <http-connector name="http-remoting-connector" connector-ref="default" security-realm="ApplicationRealm"/>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:request-controller:1.0"/>
+        <subsystem xmlns="urn:jboss:domain:security-manager:1.0">
+            <deployment-permissions>
+                <maximum-set>
+                    <permission class="java.security.AllPermission"/>
+                </maximum-set>
+            </deployment-permissions>
+        </subsystem>
+        <subsystem xmlns="urn:wildfly:elytron:1.2" final-providers="combined-providers" disallowed-providers="OracleUcrypto">
+            <providers>
+                <aggregate-providers name="combined-providers">
+                    <providers name="elytron"/>
+                    <providers name="openssl"/>
+                </aggregate-providers>
+                <provider-loader name="elytron" module="org.wildfly.security.elytron"/>
+                <provider-loader name="openssl" module="org.wildfly.openssl"/>
+            </providers>
+            <audit-logging>
+                <file-audit-log name="local-audit" path="audit.log" relative-to="jboss.server.log.dir" format="JSON"/>
+            </audit-logging>
+            <security-domains>
+                <security-domain name="ApplicationDomain" default-realm="ApplicationRealm" permission-mapper="default-permission-mapper">
+                    <realm name="ApplicationRealm" role-decoder="groups-to-roles"/>
+                    <realm name="local"/>
+                </security-domain>
+                <security-domain name="ManagementDomain" default-realm="ManagementRealm" permission-mapper="default-permission-mapper">
+                    <realm name="ManagementRealm" role-decoder="groups-to-roles"/>
+                    <realm name="local" role-mapper="super-user-mapper"/>
+                </security-domain>
+            </security-domains>
+            <security-realms>
+                <identity-realm name="local" identity="$local"/>
+                <properties-realm name="ApplicationRealm">
+                    <users-properties path="application-users.properties" relative-to="jboss.server.config.dir" digest-realm-name="ApplicationRealm"/>
+                    <groups-properties path="application-roles.properties" relative-to="jboss.server.config.dir"/>
+                </properties-realm>
+                <properties-realm name="ManagementRealm">
+                    <users-properties path="mgmt-users.properties" relative-to="jboss.server.config.dir" digest-realm-name="ManagementRealm"/>
+                    <groups-properties path="mgmt-groups.properties" relative-to="jboss.server.config.dir"/>
+                </properties-realm>
+            </security-realms>
+            <mappers>
+                <simple-permission-mapper name="default-permission-mapper" mapping-mode="first">
+                    <permission-mapping>
+                        <principal name="anonymous"/>
+                        <permission class-name="org.wildfly.extension.batch.jberet.deployment.BatchPermission" module="org.wildfly.extension.batch.jberet" target-name="*"/>
+                        <permission class-name="org.wildfly.transaction.client.RemoteTransactionPermission" module="org.wildfly.transaction.client"/>
+                        <permission class-name="org.jboss.ejb.client.RemoteEJBPermission" module="org.jboss.ejb-client"/>
+                    </permission-mapping>
+                    <permission-mapping match-all="true">
+                        <permission class-name="org.wildfly.security.auth.permission.LoginPermission"/>
+                        <permission class-name="org.wildfly.extension.batch.jberet.deployment.BatchPermission" module="org.wildfly.extension.batch.jberet" target-name="*"/>
+                        <permission class-name="org.wildfly.transaction.client.RemoteTransactionPermission" module="org.wildfly.transaction.client"/>
+                        <permission class-name="org.jboss.ejb.client.RemoteEJBPermission" module="org.jboss.ejb-client"/>
+                    </permission-mapping>
+                </simple-permission-mapper>
+                <constant-realm-mapper name="local" realm-name="local"/>
+                <simple-role-decoder name="groups-to-roles" attribute="groups"/>
+                <constant-role-mapper name="super-user-mapper">
+                    <role name="SuperUser"/>
+                </constant-role-mapper>
+            </mappers>
+            <http>
+                <http-authentication-factory name="management-http-authentication" http-server-mechanism-factory="global" security-domain="ManagementDomain">
+                    <mechanism-configuration>
+                        <mechanism mechanism-name="DIGEST">
+                            <mechanism-realm realm-name="ManagementRealm"/>
+                        </mechanism>
+                    </mechanism-configuration>
+                </http-authentication-factory>
+                <http-authentication-factory name="application-http-authentication" http-server-mechanism-factory="global" security-domain="ApplicationDomain">
+                    <mechanism-configuration>
+                        <mechanism mechanism-name="BASIC">
+                            <mechanism-realm realm-name="Application Realm"/>
+                        </mechanism>
+                        <mechanism mechanism-name="FORM"/>
+                    </mechanism-configuration>
+                </http-authentication-factory>
+                <provider-http-server-mechanism-factory name="global"/>
+            </http>
+            <sasl>
+                <sasl-authentication-factory name="management-sasl-authentication" sasl-server-factory="configured" security-domain="ManagementDomain">
+                    <mechanism-configuration>
+                        <mechanism mechanism-name="JBOSS-LOCAL-USER" realm-mapper="local"/>
+                        <mechanism mechanism-name="DIGEST-MD5">
+                            <mechanism-realm realm-name="ManagementRealm"/>
+                        </mechanism>
+                    </mechanism-configuration>
+                </sasl-authentication-factory>
+                <sasl-authentication-factory name="application-sasl-authentication" sasl-server-factory="configured" security-domain="ApplicationDomain">
+                    <mechanism-configuration>
+                        <mechanism mechanism-name="JBOSS-LOCAL-USER" realm-mapper="local"/>
+                        <mechanism mechanism-name="DIGEST-MD5">
+                            <mechanism-realm realm-name="ApplicationRealm"/>
+                        </mechanism>
+                    </mechanism-configuration>
+                </sasl-authentication-factory>
+                <configurable-sasl-server-factory name="configured" sasl-server-factory="elytron">
+                    <properties>
+                        <property name="wildfly.sasl.local-user.default-user" value="$local"/>
+                    </properties>
+                </configurable-sasl-server-factory>
+                <mechanism-provider-filtering-sasl-server-factory name="elytron" sasl-server-factory="global">
+                    <filters>
+                        <filter provider-name="WildFlyElytron"/>
+                    </filters>
+                </mechanism-provider-filtering-sasl-server-factory>
+                <provider-sasl-server-factory name="global"/>
+            </sasl>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:security:2.0">
+            <security-domains>
+                <security-domain name="other" cache-type="default">
+                    <authentication>
+                        <login-module code="Remoting" flag="optional">
+                            <module-option name="password-stacking" value="useFirstPass"/>
+                        </login-module>
+                        <login-module code="RealmDirect" flag="required">
+                            <module-option name="password-stacking" value="useFirstPass"/>
+                        </login-module>
+                    </authentication>
+                </security-domain>
+                <security-domain name="jboss-web-policy" cache-type="default">
+                    <authorization>
+                        <policy-module code="Delegating" flag="required"/>
+                    </authorization>
+                </security-domain>
+                <security-domain name="jboss-ejb-policy" cache-type="default">
+                    <authorization>
+                        <policy-module code="Delegating" flag="required"/>
+                    </authorization>
+                </security-domain>
+                <security-domain name="jaspitest" cache-type="default">
+                    <authentication-jaspi>
+                        <login-module-stack name="dummy">
+                            <login-module code="Dummy" flag="optional"/>
+                        </login-module-stack>
+                        <auth-module code="Dummy"/>
+                    </authentication-jaspi>
+                </security-domain>
+            </security-domains>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:transactions:4.0">
+            <core-environment>
+                <process-id>
+                    <uuid/>
+                </process-id>
+            </core-environment>
+            <recovery-environment socket-binding="txn-recovery-environment" status-socket-binding="txn-status-manager"/>
+            <object-store path="tx-object-store" relative-to="jboss.server.data.dir"/>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:undertow:4.0">
+            <buffer-cache name="default"/>
+            <server name="default-server">
+                <http-listener name="default" socket-binding="http" redirect-socket="https" enable-http2="true"/>
+                <https-listener name="https" socket-binding="https" security-realm="ApplicationRealm" enable-http2="true"/>
+                <host name="default-host" alias="localhost">
+                    <location name="/" handler="welcome-content"/>
+                    <http-invoker security-realm="ApplicationRealm"/>
+                </host>
+            </server>
+            <servlet-container name="default">
+                <jsp-config/>
+                <websockets/>
+            </servlet-container>
+            <handlers>
+                <file name="welcome-content" path="${jboss.home.dir}/welcome-content"/>
+            </handlers>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:keycloak-server:1.1">
+            <web-context>auth</web-context>
+            <providers>
+                <provider>classpath:${jboss.home.dir}/providers/*</provider>
+            </providers>
+            <master-realm-name>master</master-realm-name>
+            <scheduled-task-interval>900</scheduled-task-interval>
+            <theme>
+                <staticMaxAge>2592000</staticMaxAge>
+                <cacheThemes>true</cacheThemes>
+                <cacheTemplates>true</cacheTemplates>
+                <dir>${jboss.home.dir}/themes</dir>
+            </theme>
+            <spi name="eventsStore">
+                <provider name="jpa" enabled="true">
+                    <properties>
+                        <property name="exclude-events" value="[&quot;REFRESH_TOKEN&quot;]"/>
+                    </properties>
+                </provider>
+            </spi>
+            <spi name="userCache">
+                <provider name="default" enabled="true"/>
+            </spi>
+            <spi name="userSessionPersister">
+                <default-provider>jpa</default-provider>
+            </spi>
+            <spi name="timer">
+                <default-provider>basic</default-provider>
+            </spi>
+            <spi name="connectionsHttpClient">
+                <provider name="default" enabled="true"/>
+            </spi>
+            <spi name="connectionsJpa">
+                <provider name="default" enabled="true">
+                    <properties>
+                        <property name="dataSource" value="java:jboss/datasources/KeycloakDS"/>
+                        <property name="initializeEmpty" value="true"/>
+                        <property name="migrationStrategy" value="update"/>
+                        <property name="migrationExport" value="${jboss.home.dir}/keycloak-database-update.sql"/>
+                    </properties>
+                </provider>
+            </spi>
+            <spi name="realmCache">
+                <provider name="default" enabled="true"/>
+            </spi>
+            <spi name="connectionsInfinispan">
+                <default-provider>default</default-provider>
+                <provider name="default" enabled="true">
+                    <properties>
+                        <property name="cacheContainer" value="java:comp/env/infinispan/Keycloak"/>
+                    </properties>
+                </provider>
+            </spi>
+            <spi name="jta-lookup">
+                <default-provider>${keycloak.jta.lookup.provider:jboss}</default-provider>
+                <provider name="jboss" enabled="true"/>
+            </spi>
+            <spi name="publicKeyStorage">
+                <provider name="infinispan" enabled="true">
+                    <properties>
+                        <property name="minTimeBetweenRequests" value="10"/>
+                    </properties>
+                </provider>
+            </spi>
+            <spi name="x509cert-lookup">
+                <default-provider>${keycloak.x509cert.lookup.provider:default}</default-provider>
+                <provider name="default" enabled="true"/>
+            </spi>
+        </subsystem>
+    </profile>
+    <interfaces>
+        <interface name="management">
+            <inet-address value="${jboss.bind.address.management:127.0.0.1}"/>
+        </interface>
+        <interface name="public">
+            <inet-address value="${jboss.bind.address:127.0.0.1}"/>
+        </interface>
+    </interfaces>
+    <socket-binding-group name="standard-sockets" default-interface="public" port-offset="${jboss.socket.binding.port-offset:0}">
+        <socket-binding name="management-http" interface="management" port="${jboss.management.http.port:9990}"/>
+        <socket-binding name="management-https" interface="management" port="${jboss.management.https.port:9993}"/>
+        <socket-binding name="ajp" port="${jboss.ajp.port:8009}"/>
+        <socket-binding name="http" port="${jboss.http.port:8080}"/>
+        <socket-binding name="https" port="${jboss.https.port:8443}"/>
+        <socket-binding name="txn-recovery-environment" port="4712"/>
+        <socket-binding name="txn-status-manager" port="4713"/>
+        <outbound-socket-binding name="mail-smtp">
+            <remote-destination host="localhost" port="25"/>
+        </outbound-socket-binding>
+    </socket-binding-group>
+</server>

testsuite/integration-arquillian/tests/other/server-config-migration/src/test/resources/standalone/standalone-ha-3.4.3.Final-redhat-2.xml

@@ -0,0 +1,631 @@
+<?xml version='1.0' encoding='UTF-8'?>
+
+<server xmlns="urn:jboss:domain:5.0">
+    <extensions>
+        <extension module="org.jboss.as.clustering.infinispan"/>
+        <extension module="org.jboss.as.clustering.jgroups"/>
+        <extension module="org.jboss.as.connector"/>
+        <extension module="org.jboss.as.deployment-scanner"/>
+        <extension module="org.jboss.as.ee"/>
+        <extension module="org.jboss.as.ejb3"/>
+        <extension module="org.jboss.as.jaxrs"/>
+        <extension module="org.jboss.as.jmx"/>
+        <extension module="org.jboss.as.jpa"/>
+        <extension module="org.jboss.as.logging"/>
+        <extension module="org.jboss.as.mail"/>
+        <extension module="org.jboss.as.modcluster"/>
+        <extension module="org.jboss.as.naming"/>
+        <extension module="org.jboss.as.remoting"/>
+        <extension module="org.jboss.as.security"/>
+        <extension module="org.jboss.as.transactions"/>
+        <extension module="org.keycloak.keycloak-server-subsystem"/>
+        <extension module="org.wildfly.extension.bean-validation"/>
+        <extension module="org.wildfly.extension.elytron"/>
+        <extension module="org.wildfly.extension.io"/>
+        <extension module="org.wildfly.extension.request-controller"/>
+        <extension module="org.wildfly.extension.security.manager"/>
+        <extension module="org.wildfly.extension.undertow"/>
+    </extensions>
+    <management>
+        <security-realms>
+            <security-realm name="ManagementRealm">
+                <authentication>
+                    <local default-user="$local" skip-group-loading="true"/>
+                    <properties path="mgmt-users.properties" relative-to="jboss.server.config.dir"/>
+                </authentication>
+                <authorization map-groups-to-roles="false">
+                    <properties path="mgmt-groups.properties" relative-to="jboss.server.config.dir"/>
+                </authorization>
+            </security-realm>
+            <security-realm name="ApplicationRealm">
+                <server-identities>
+                    <ssl>
+                        <keystore path="application.keystore" relative-to="jboss.server.config.dir" keystore-password="password" alias="server" key-password="password" generate-self-signed-certificate-host="localhost"/>
+                    </ssl>
+                </server-identities>
+                <authentication>
+                    <local default-user="$local" allowed-users="*" skip-group-loading="true"/>
+                    <properties path="application-users.properties" relative-to="jboss.server.config.dir"/>
+                </authentication>
+                <authorization>
+                    <properties path="application-roles.properties" relative-to="jboss.server.config.dir"/>
+                </authorization>
+            </security-realm>
+        </security-realms>
+        <audit-log>
+            <formatters>
+                <json-formatter name="json-formatter"/>
+            </formatters>
+            <handlers>
+                <file-handler name="file" formatter="json-formatter" path="audit-log.log" relative-to="jboss.server.data.dir"/>
+            </handlers>
+            <logger log-boot="true" log-read-only="false" enabled="false">
+                <handlers>
+                    <handler name="file"/>
+                </handlers>
+            </logger>
+        </audit-log>
+        <management-interfaces>
+            <http-interface security-realm="ManagementRealm">
+                <http-upgrade enabled="true"/>
+                <socket-binding http="management-http"/>
+            </http-interface>
+        </management-interfaces>
+        <access-control provider="simple">
+            <role-mapping>
+                <role name="SuperUser">
+                    <include>
+                        <user name="$local"/>
+                    </include>
+                </role>
+            </role-mapping>
+        </access-control>
+    </management>
+    <profile>
+        <subsystem xmlns="urn:jboss:domain:logging:3.0">
+            <console-handler name="CONSOLE">
+                <level name="INFO"/>
+                <formatter>
+                    <named-formatter name="COLOR-PATTERN"/>
+                </formatter>
+            </console-handler>
+            <periodic-rotating-file-handler name="FILE" autoflush="true">
+                <formatter>
+                    <named-formatter name="PATTERN"/>
+                </formatter>
+                <file relative-to="jboss.server.log.dir" path="server.log"/>
+                <suffix value=".yyyy-MM-dd"/>
+                <append value="true"/>
+            </periodic-rotating-file-handler>
+            <logger category="com.arjuna">
+                <level name="WARN"/>
+            </logger>
+            <logger category="org.jboss.as.config">
+                <level name="DEBUG"/>
+            </logger>
+            <logger category="sun.rmi">
+                <level name="WARN"/>
+            </logger>
+            <root-logger>
+                <level name="INFO"/>
+                <handlers>
+                    <handler name="CONSOLE"/>
+                    <handler name="FILE"/>
+                </handlers>
+            </root-logger>
+            <formatter name="PATTERN">
+                <pattern-formatter pattern="%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%c] (%t) %s%e%n"/>
+            </formatter>
+            <formatter name="COLOR-PATTERN">
+                <pattern-formatter pattern="%K{level}%d{HH:mm:ss,SSS} %-5p [%c] (%t) %s%e%n"/>
+            </formatter>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:bean-validation:1.0"/>
+        <subsystem xmlns="urn:jboss:domain:datasources:5.0">
+            <datasources>
+                <datasource jndi-name="java:jboss/datasources/ExampleDS" pool-name="ExampleDS" enabled="true" use-java-context="true">
+                    <connection-url>jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE</connection-url>
+                    <driver>h2</driver>
+                    <security>
+                        <user-name>sa</user-name>
+                        <password>sa</password>
+                    </security>
+                </datasource>
+                <datasource jndi-name="java:jboss/datasources/KeycloakDS" pool-name="KeycloakDS" enabled="true" use-java-context="true">
+                    <connection-url>jdbc:h2:${jboss.server.data.dir}/keycloak;AUTO_SERVER=TRUE</connection-url>
+                    <driver>h2</driver>
+                    <security>
+                        <user-name>sa</user-name>
+                        <password>sa</password>
+                    </security>
+                </datasource>
+                <drivers>
+                    <driver name="h2" module="com.h2database.h2">
+                        <xa-datasource-class>org.h2.jdbcx.JdbcDataSource</xa-datasource-class>
+                    </driver>
+                </drivers>
+            </datasources>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:deployment-scanner:2.0">
+            <deployment-scanner path="deployments" relative-to="jboss.server.base.dir" scan-interval="5000" runtime-failure-causes-rollback="${jboss.deployment.scanner.rollback.on.failure:false}"/>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:ee:4.0">
+            <spec-descriptor-property-replacement>false</spec-descriptor-property-replacement>
+            <concurrent>
+                <context-services>
+                    <context-service name="default" jndi-name="java:jboss/ee/concurrency/context/default" use-transaction-setup-provider="true"/>
+                </context-services>
+                <managed-thread-factories>
+                    <managed-thread-factory name="default" jndi-name="java:jboss/ee/concurrency/factory/default" context-service="default"/>
+                </managed-thread-factories>
+                <managed-executor-services>
+                    <managed-executor-service name="default" jndi-name="java:jboss/ee/concurrency/executor/default" context-service="default" hung-task-threshold="60000" keepalive-time="5000"/>
+                </managed-executor-services>
+                <managed-scheduled-executor-services>
+                    <managed-scheduled-executor-service name="default" jndi-name="java:jboss/ee/concurrency/scheduler/default" context-service="default" hung-task-threshold="60000" keepalive-time="3000"/>
+                </managed-scheduled-executor-services>
+            </concurrent>
+            <default-bindings context-service="java:jboss/ee/concurrency/context/default" datasource="java:jboss/datasources/ExampleDS" managed-executor-service="java:jboss/ee/concurrency/executor/default" managed-scheduled-executor-service="java:jboss/ee/concurrency/scheduler/default" managed-thread-factory="java:jboss/ee/concurrency/factory/default"/>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:ejb3:5.0">
+            <session-bean>
+                <stateless>
+                    <bean-instance-pool-ref pool-name="slsb-strict-max-pool"/>
+                </stateless>
+                <stateful default-access-timeout="5000" cache-ref="distributable" passivation-disabled-cache-ref="simple"/>
+                <singleton default-access-timeout="5000"/>
+            </session-bean>
+            <pools>
+                <bean-instance-pools>
+                    <strict-max-pool name="slsb-strict-max-pool" derive-size="from-worker-pools" instance-acquisition-timeout="5" instance-acquisition-timeout-unit="MINUTES"/>
+                    <strict-max-pool name="mdb-strict-max-pool" derive-size="from-cpu-count" instance-acquisition-timeout="5" instance-acquisition-timeout-unit="MINUTES"/>
+                </bean-instance-pools>
+            </pools>
+            <caches>
+                <cache name="simple"/>
+                <cache name="distributable" passivation-store-ref="infinispan" aliases="passivating clustered"/>
+            </caches>
+            <passivation-stores>
+                <passivation-store name="infinispan" cache-container="ejb" max-size="10000"/>
+            </passivation-stores>
+            <async thread-pool-name="default"/>
+            <timer-service thread-pool-name="default" default-data-store="default-file-store">
+                <data-stores>
+                    <file-data-store name="default-file-store" path="timer-service-data" relative-to="jboss.server.data.dir"/>
+                </data-stores>
+            </timer-service>
+            <remote connector-ref="http-remoting-connector" thread-pool-name="default">
+                <channel-creation-options>
+                    <option name="READ_TIMEOUT" value="${prop.remoting-connector.read.timeout:20}" type="xnio"/>
+                    <option name="MAX_OUTBOUND_MESSAGES" value="1234" type="remoting"/>
+                </channel-creation-options>
+            </remote>
+            <thread-pools>
+                <thread-pool name="default">
+                    <max-threads count="10"/>
+                    <keepalive-time time="100" unit="milliseconds"/>
+                </thread-pool>
+            </thread-pools>
+            <default-security-domain value="other"/>
+            <default-missing-method-permissions-deny-access value="true"/>
+            <log-system-exceptions value="true"/>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:io:2.0">
+            <worker name="default"/>
+            <buffer-pool name="default"/>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:infinispan:4.0">
+            <cache-container name="keycloak" jndi-name="infinispan/Keycloak">
+                <transport lock-timeout="60000"/>
+                <local-cache name="realms">
+                    <eviction max-entries="10000" strategy="LRU"/>
+                </local-cache>
+                <local-cache name="users">
+                    <eviction max-entries="10000" strategy="LRU"/>
+                </local-cache>
+                <distributed-cache name="sessions" mode="SYNC" owners="1"/>
+                <distributed-cache name="authenticationSessions" mode="SYNC" owners="1"/>
+                <distributed-cache name="offlineSessions" mode="SYNC" owners="1"/>
+                <distributed-cache name="clientSessions" mode="SYNC" owners="1"/>
+                <distributed-cache name="offlineClientSessions" mode="SYNC" owners="1"/>
+                <distributed-cache name="loginFailures" mode="SYNC" owners="1"/>
+                <local-cache name="authorization">
+                    <eviction max-entries="10000" strategy="LRU"/>
+                </local-cache>
+                <replicated-cache name="work" mode="SYNC"/>
+                <local-cache name="keys">
+                    <eviction max-entries="1000" strategy="LRU"/>
+                    <expiration max-idle="3600000"/>
+                </local-cache>
+                <distributed-cache name="actionTokens" mode="SYNC" owners="2">
+                    <eviction max-entries="-1" strategy="NONE"/>
+                    <expiration max-idle="-1" interval="300000"/>
+                </distributed-cache>
+            </cache-container>
+            <cache-container name="server" aliases="singleton cluster" default-cache="default" module="org.wildfly.clustering.server">
+                <transport lock-timeout="60000"/>
+                <replicated-cache name="default">
+                    <transaction mode="BATCH"/>
+                </replicated-cache>
+            </cache-container>
+            <cache-container name="web" default-cache="dist" module="org.wildfly.clustering.web.infinispan">
+                <transport lock-timeout="60000"/>
+                <distributed-cache name="dist">
+                    <locking isolation="REPEATABLE_READ"/>
+                    <transaction mode="BATCH"/>
+                    <file-store/>
+                </distributed-cache>
+            </cache-container>
+            <cache-container name="ejb" aliases="sfsb" default-cache="dist" module="org.wildfly.clustering.ejb.infinispan">
+                <transport lock-timeout="60000"/>
+                <distributed-cache name="dist">
+                    <locking isolation="REPEATABLE_READ"/>
+                    <transaction mode="BATCH"/>
+                    <file-store/>
+                </distributed-cache>
+            </cache-container>
+            <cache-container name="hibernate" default-cache="local-query" module="org.hibernate.infinispan">
+                <transport lock-timeout="60000"/>
+                <local-cache name="local-query">
+                    <eviction strategy="LRU" max-entries="10000"/>
+                    <expiration max-idle="100000"/>
+                </local-cache>
+                <invalidation-cache name="entity">
+                    <transaction mode="NON_XA"/>
+                    <eviction strategy="LRU" max-entries="10000"/>
+                    <expiration max-idle="100000"/>
+                </invalidation-cache>
+                <replicated-cache name="timestamps" mode="ASYNC"/>
+            </cache-container>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:jaxrs:1.0"/>
+        <subsystem xmlns="urn:jboss:domain:jca:5.0">
+            <archive-validation enabled="true" fail-on-error="true" fail-on-warn="false"/>
+            <bean-validation enabled="true"/>
+            <default-workmanager>
+                <short-running-threads>
+                    <core-threads count="50"/>
+                    <queue-length count="50"/>
+                    <max-threads count="50"/>
+                    <keepalive-time time="10" unit="seconds"/>
+                </short-running-threads>
+                <long-running-threads>
+                    <core-threads count="50"/>
+                    <queue-length count="50"/>
+                    <max-threads count="50"/>
+                    <keepalive-time time="10" unit="seconds"/>
+                </long-running-threads>
+            </default-workmanager>
+            <cached-connection-manager/>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:jgroups:5.0">
+            <channels default="ee">
+                <channel name="ee" stack="udp" cluster="ejb"/>
+            </channels>
+            <stacks>
+                <stack name="udp">
+                    <transport type="UDP" socket-binding="jgroups-udp"/>
+                    <protocol type="PING"/>
+                    <protocol type="MERGE3"/>
+                    <protocol type="FD_SOCK"/>
+                    <protocol type="FD_ALL"/>
+                    <protocol type="VERIFY_SUSPECT"/>
+                    <protocol type="pbcast.NAKACK2"/>
+                    <protocol type="UNICAST3"/>
+                    <protocol type="pbcast.STABLE"/>
+                    <protocol type="pbcast.GMS"/>
+                    <protocol type="UFC"/>
+                    <protocol type="MFC"/>
+                    <protocol type="FRAG2"/>
+                </stack>
+                <stack name="tcp">
+                    <transport type="TCP" socket-binding="jgroups-tcp"/>
+                    <socket-protocol type="MPING" socket-binding="jgroups-mping"/>
+                    <protocol type="MERGE3"/>
+                    <protocol type="FD_SOCK"/>
+                    <protocol type="FD_ALL"/>
+                    <protocol type="VERIFY_SUSPECT"/>
+                    <protocol type="pbcast.NAKACK2"/>
+                    <protocol type="UNICAST3"/>
+                    <protocol type="pbcast.STABLE"/>
+                    <protocol type="pbcast.GMS"/>
+                    <protocol type="MFC"/>
+                    <protocol type="FRAG2"/>
+                </stack>
+            </stacks>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:jmx:1.3">
+            <expose-resolved-model/>
+            <expose-expression-model/>
+            <remoting-connector/>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:jpa:1.1">
+            <jpa default-datasource="" default-extended-persistence-inheritance="DEEP"/>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:mail:3.0">
+            <mail-session name="default" jndi-name="java:jboss/mail/Default">
+                <smtp-server outbound-socket-binding-ref="mail-smtp"/>
+            </mail-session>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:modcluster:3.0">
+            <mod-cluster-config advertise-socket="modcluster" connector="ajp">
+                <dynamic-load-provider>
+                    <load-metric type="cpu"/>
+                </dynamic-load-provider>
+            </mod-cluster-config>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:naming:2.0">
+            <remote-naming/>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:remoting:4.0">
+            <endpoint/>
+            <http-connector name="http-remoting-connector" connector-ref="default" security-realm="ApplicationRealm"/>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:request-controller:1.0"/>
+        <subsystem xmlns="urn:jboss:domain:security-manager:1.0">
+            <deployment-permissions>
+                <maximum-set>
+                    <permission class="java.security.AllPermission"/>
+                </maximum-set>
+            </deployment-permissions>
+        </subsystem>
+        <subsystem xmlns="urn:wildfly:elytron:1.2" final-providers="combined-providers" disallowed-providers="OracleUcrypto">
+            <providers>
+                <aggregate-providers name="combined-providers">
+                    <providers name="elytron"/>
+                    <providers name="openssl"/>
+                </aggregate-providers>
+                <provider-loader name="elytron" module="org.wildfly.security.elytron"/>
+                <provider-loader name="openssl" module="org.wildfly.openssl"/>
+            </providers>
+            <audit-logging>
+                <file-audit-log name="local-audit" path="audit.log" relative-to="jboss.server.log.dir" format="JSON"/>
+            </audit-logging>
+            <security-domains>
+                <security-domain name="ApplicationDomain" default-realm="ApplicationRealm" permission-mapper="default-permission-mapper">
+                    <realm name="ApplicationRealm" role-decoder="groups-to-roles"/>
+                    <realm name="local"/>
+                </security-domain>
+                <security-domain name="ManagementDomain" default-realm="ManagementRealm" permission-mapper="default-permission-mapper">
+                    <realm name="ManagementRealm" role-decoder="groups-to-roles"/>
+                    <realm name="local" role-mapper="super-user-mapper"/>
+                </security-domain>
+            </security-domains>
+            <security-realms>
+                <identity-realm name="local" identity="$local"/>
+                <properties-realm name="ApplicationRealm">
+                    <users-properties path="application-users.properties" relative-to="jboss.server.config.dir" digest-realm-name="ApplicationRealm"/>
+                    <groups-properties path="application-roles.properties" relative-to="jboss.server.config.dir"/>
+                </properties-realm>
+                <properties-realm name="ManagementRealm">
+                    <users-properties path="mgmt-users.properties" relative-to="jboss.server.config.dir" digest-realm-name="ManagementRealm"/>
+                    <groups-properties path="mgmt-groups.properties" relative-to="jboss.server.config.dir"/>
+                </properties-realm>
+            </security-realms>
+            <mappers>
+                <simple-permission-mapper name="default-permission-mapper" mapping-mode="first">
+                    <permission-mapping>
+                        <principal name="anonymous"/>
+                        <permission class-name="org.wildfly.extension.batch.jberet.deployment.BatchPermission" module="org.wildfly.extension.batch.jberet" target-name="*"/>
+                        <permission class-name="org.wildfly.transaction.client.RemoteTransactionPermission" module="org.wildfly.transaction.client"/>
+                        <permission class-name="org.jboss.ejb.client.RemoteEJBPermission" module="org.jboss.ejb-client"/>
+                    </permission-mapping>
+                    <permission-mapping match-all="true">
+                        <permission class-name="org.wildfly.security.auth.permission.LoginPermission"/>
+                        <permission class-name="org.wildfly.extension.batch.jberet.deployment.BatchPermission" module="org.wildfly.extension.batch.jberet" target-name="*"/>
+                        <permission class-name="org.wildfly.transaction.client.RemoteTransactionPermission" module="org.wildfly.transaction.client"/>
+                        <permission class-name="org.jboss.ejb.client.RemoteEJBPermission" module="org.jboss.ejb-client"/>
+                    </permission-mapping>
+                </simple-permission-mapper>
+                <constant-realm-mapper name="local" realm-name="local"/>
+                <simple-role-decoder name="groups-to-roles" attribute="groups"/>
+                <constant-role-mapper name="super-user-mapper">
+                    <role name="SuperUser"/>
+                </constant-role-mapper>
+            </mappers>
+            <http>
+                <http-authentication-factory name="management-http-authentication" http-server-mechanism-factory="global" security-domain="ManagementDomain">
+                    <mechanism-configuration>
+                        <mechanism mechanism-name="DIGEST">
+                            <mechanism-realm realm-name="ManagementRealm"/>
+                        </mechanism>
+                    </mechanism-configuration>
+                </http-authentication-factory>
+                <http-authentication-factory name="application-http-authentication" http-server-mechanism-factory="global" security-domain="ApplicationDomain">
+                    <mechanism-configuration>
+                        <mechanism mechanism-name="BASIC">
+                            <mechanism-realm realm-name="Application Realm"/>
+                        </mechanism>
+                        <mechanism mechanism-name="FORM"/>
+                    </mechanism-configuration>
+                </http-authentication-factory>
+                <provider-http-server-mechanism-factory name="global"/>
+            </http>
+            <sasl>
+                <sasl-authentication-factory name="management-sasl-authentication" sasl-server-factory="configured" security-domain="ManagementDomain">
+                    <mechanism-configuration>
+                        <mechanism mechanism-name="JBOSS-LOCAL-USER" realm-mapper="local"/>
+                        <mechanism mechanism-name="DIGEST-MD5">
+                            <mechanism-realm realm-name="ManagementRealm"/>
+                        </mechanism>
+                    </mechanism-configuration>
+                </sasl-authentication-factory>
+                <sasl-authentication-factory name="application-sasl-authentication" sasl-server-factory="configured" security-domain="ApplicationDomain">
+                    <mechanism-configuration>
+                        <mechanism mechanism-name="JBOSS-LOCAL-USER" realm-mapper="local"/>
+                        <mechanism mechanism-name="DIGEST-MD5">
+                            <mechanism-realm realm-name="ApplicationRealm"/>
+                        </mechanism>
+                    </mechanism-configuration>
+                </sasl-authentication-factory>
+                <configurable-sasl-server-factory name="configured" sasl-server-factory="elytron">
+                    <properties>
+                        <property name="wildfly.sasl.local-user.default-user" value="$local"/>
+                    </properties>
+                </configurable-sasl-server-factory>
+                <mechanism-provider-filtering-sasl-server-factory name="elytron" sasl-server-factory="global">
+                    <filters>
+                        <filter provider-name="WildFlyElytron"/>
+                    </filters>
+                </mechanism-provider-filtering-sasl-server-factory>
+                <provider-sasl-server-factory name="global"/>
+            </sasl>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:security:2.0">
+            <security-domains>
+                <security-domain name="other" cache-type="default">
+                    <authentication>
+                        <login-module code="Remoting" flag="optional">
+                            <module-option name="password-stacking" value="useFirstPass"/>
+                        </login-module>
+                        <login-module code="RealmDirect" flag="required">
+                            <module-option name="password-stacking" value="useFirstPass"/>
+                        </login-module>
+                    </authentication>
+                </security-domain>
+                <security-domain name="jboss-web-policy" cache-type="default">
+                    <authorization>
+                        <policy-module code="Delegating" flag="required"/>
+                    </authorization>
+                </security-domain>
+                <security-domain name="jboss-ejb-policy" cache-type="default">
+                    <authorization>
+                        <policy-module code="Delegating" flag="required"/>
+                    </authorization>
+                </security-domain>
+                <security-domain name="jaspitest" cache-type="default">
+                    <authentication-jaspi>
+                        <login-module-stack name="dummy">
+                            <login-module code="Dummy" flag="optional"/>
+                        </login-module-stack>
+                        <auth-module code="Dummy"/>
+                    </authentication-jaspi>
+                </security-domain>
+            </security-domains>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:transactions:4.0">
+            <core-environment>
+                <process-id>
+                    <uuid/>
+                </process-id>
+            </core-environment>
+            <recovery-environment socket-binding="txn-recovery-environment" status-socket-binding="txn-status-manager"/>
+            <object-store path="tx-object-store" relative-to="jboss.server.data.dir"/>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:undertow:4.0">
+            <buffer-cache name="default"/>
+            <server name="default-server">
+                <ajp-listener name="ajp" socket-binding="ajp"/>
+                <http-listener name="default" socket-binding="http" redirect-socket="https" enable-http2="true"/>
+                <https-listener name="https" socket-binding="https" security-realm="ApplicationRealm" enable-http2="true"/>
+                <host name="default-host" alias="localhost">
+                    <location name="/" handler="welcome-content"/>
+                    <http-invoker security-realm="ApplicationRealm"/>
+                </host>
+            </server>
+            <servlet-container name="default">
+                <jsp-config/>
+                <websockets/>
+            </servlet-container>
+            <handlers>
+                <file name="welcome-content" path="${jboss.home.dir}/welcome-content"/>
+            </handlers>
+        </subsystem>
+        <subsystem xmlns="urn:jboss:domain:keycloak-server:1.1">
+            <web-context>auth</web-context>
+            <providers>
+                <provider>classpath:${jboss.home.dir}/providers/*</provider>
+            </providers>
+            <master-realm-name>master</master-realm-name>
+            <scheduled-task-interval>900</scheduled-task-interval>
+            <theme>
+                <staticMaxAge>2592000</staticMaxAge>
+                <cacheThemes>true</cacheThemes>
+                <cacheTemplates>true</cacheTemplates>
+                <dir>${jboss.home.dir}/themes</dir>
+            </theme>
+            <spi name="eventsStore">
+                <provider name="jpa" enabled="true">
+                    <properties>
+                        <property name="exclude-events" value="[&quot;REFRESH_TOKEN&quot;]"/>
+                    </properties>
+                </provider>
+            </spi>
+            <spi name="userCache">
+                <provider name="default" enabled="true"/>
+            </spi>
+            <spi name="userSessionPersister">
+                <default-provider>jpa</default-provider>
+            </spi>
+            <spi name="timer">
+                <default-provider>basic</default-provider>
+            </spi>
+            <spi name="connectionsHttpClient">
+                <provider name="default" enabled="true"/>
+            </spi>
+            <spi name="connectionsJpa">
+                <provider name="default" enabled="true">
+                    <properties>
+                        <property name="dataSource" value="java:jboss/datasources/KeycloakDS"/>
+                        <property name="initializeEmpty" value="true"/>
+                        <property name="migrationStrategy" value="update"/>
+                        <property name="migrationExport" value="${jboss.home.dir}/keycloak-database-update.sql"/>
+                    </properties>
+                </provider>
+            </spi>
+            <spi name="realmCache">
+                <provider name="default" enabled="true"/>
+            </spi>
+            <spi name="connectionsInfinispan">
+                <default-provider>default</default-provider>
+                <provider name="default" enabled="true">
+                    <properties>
+                        <property name="cacheContainer" value="java:comp/env/infinispan/Keycloak"/>
+                    </properties>
+                </provider>
+            </spi>
+            <spi name="jta-lookup">
+                <default-provider>${keycloak.jta.lookup.provider:jboss}</default-provider>
+                <provider name="jboss" enabled="true"/>
+            </spi>
+            <spi name="publicKeyStorage">
+                <provider name="infinispan" enabled="true">
+                    <properties>
+                        <property name="minTimeBetweenRequests" value="10"/>
+                    </properties>
+                </provider>
+            </spi>
+            <spi name="x509cert-lookup">
+                <default-provider>${keycloak.x509cert.lookup.provider:default}</default-provider>
+                <provider name="default" enabled="true"/>
+            </spi>
+        </subsystem>
+    </profile>
+    <interfaces>
+        <interface name="management">
+            <inet-address value="${jboss.bind.address.management:127.0.0.1}"/>
+        </interface>
+        <interface name="public">
+            <inet-address value="${jboss.bind.address:127.0.0.1}"/>
+        </interface>
+        <interface name="private">
+            <inet-address value="${jboss.bind.address.private:127.0.0.1}"/>
+        </interface>
+    </interfaces>
+    <socket-binding-group name="standard-sockets" default-interface="public" port-offset="${jboss.socket.binding.port-offset:0}">
+        <socket-binding name="management-http" interface="management" port="${jboss.management.http.port:9990}"/>
+        <socket-binding name="management-https" interface="management" port="${jboss.management.https.port:9993}"/>
+        <socket-binding name="ajp" port="${jboss.ajp.port:8009}"/>
+        <socket-binding name="http" port="${jboss.http.port:8080}"/>
+        <socket-binding name="https" port="${jboss.https.port:8443}"/>
+        <socket-binding name="jgroups-mping" interface="private" port="0" multicast-address="${jboss.default.multicast.address:230.0.0.4}" multicast-port="45700"/>
+        <socket-binding name="jgroups-tcp" interface="private" port="7600"/>
+        <socket-binding name="jgroups-udp" interface="private" port="55200" multicast-address="${jboss.default.multicast.address:230.0.0.4}" multicast-port="45688"/>
+        <socket-binding name="modcluster" port="0" multicast-address="${jboss.modcluster.multicast.address:224.0.1.105}" multicast-port="23364"/>
+        <socket-binding name="txn-recovery-environment" port="4712"/>
+        <socket-binding name="txn-status-manager" port="4713"/>
+        <outbound-socket-binding name="mail-smtp">
+            <remote-destination host="localhost" port="25"/>
+        </outbound-socket-binding>
+    </socket-binding-group>
+</server>

testsuite/integration-arquillian/tests/pom.xml

@@ -950,7 +950,7 @@
                 <cache.server.jboss>true</cache.server.jboss>
                 <cache.server.config.dir>${cache.server.home}/standalone/configuration</cache.server.config.dir>
                 <keycloak.testsuite.logging.pattern>%d{HH:mm:ss,SSS} [%t] %-5p [%c{1.}] %m%n</keycloak.testsuite.logging.pattern>
-                <keycloak.connectionsInfinispan.default.remoteStoreSecurityEnabled>true</keycloak.connectionsInfinispan.default.remoteStoreSecurityEnabled>
+                <keycloak.connectionsInfinispan.default.remoteStoreSecurityEnabled>false</keycloak.connectionsInfinispan.default.remoteStoreSecurityEnabled>
             </properties>
             <dependencies>
                 <dependency>