KEYCLOAK-4626 Support for sticky sessions with AUTH_SESSION_ID cookie. Clustering tests with embedded undertow. Last fixes.

distribution/feature-packs/server-feature-pack/src/main/resources/modules/system/layers/keycloak/org/keycloak/keycloak-model-infinispan/main/module.xml

@@ -33,6 +33,7 @@
         <module name="org.infinispan.commons"/>
         <module name="org.infinispan.cachestore.remote"/>
         <module name="org.infinispan.client.hotrod"/>
+        <module name="org.jgroups"/>
         <module name="org.jboss.logging"/>
         <module name="javax.api"/>
     </dependencies>

integration/admin-client/src/main/java/org/keycloak/admin/client/resource/ClientPoliciesResource.java

@@ -27,6 +27,7 @@ import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 
 import org.jboss.resteasy.annotations.cache.NoCache;
+import org.keycloak.representations.idm.authorization.AbstractPolicyRepresentation;
 import org.keycloak.representations.idm.authorization.ClientPolicyRepresentation;
 
 /**

misc/Testsuite.md

@@ -132,6 +132,12 @@ kinit hnelson@KEYCLOAK.ORG
 and provide password `secret`
 
 Now when you access `http://localhost:8081/auth/realms/master/account` you should be logged in automatically as user `hnelson` .
+
+Simple loadbalancer
+-------------------
+
+You can run class `SimpleUndertowLoadBalancer` from IDE. By default, it executes the embedded undertow loadbalancer running on `http://localhost:8180`, which communicates with 2 backend Keycloak nodes 
+running on `http://localhost:8181` and `http://localhost:8182` . See javadoc for more details.
  
 
 Create many users or offline sessions

model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java

@@ -19,6 +19,8 @@ package org.keycloak.connections.infinispan;
 
 import java.util.concurrent.TimeUnit;
 
+import org.infinispan.commons.util.FileLookup;
+import org.infinispan.commons.util.FileLookupFactory;
 import org.infinispan.configuration.cache.CacheMode;
 import org.infinispan.configuration.cache.Configuration;
 import org.infinispan.configuration.cache.ConfigurationBuilder;
@@ -28,10 +30,12 @@ import org.infinispan.eviction.EvictionType;
 import org.infinispan.manager.DefaultCacheManager;
 import org.infinispan.manager.EmbeddedCacheManager;
 import org.infinispan.persistence.remote.configuration.RemoteStoreConfigurationBuilder;
+import org.infinispan.remoting.transport.jgroups.JGroupsTransport;
 import org.infinispan.transaction.LockingMode;
 import org.infinispan.transaction.TransactionMode;
 import org.infinispan.transaction.lookup.DummyTransactionManagerLookup;
 import org.jboss.logging.Logger;
+import org.jgroups.JChannel;
 import org.keycloak.Config;
 import org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory;
 import org.keycloak.models.KeycloakSession;
@@ -139,7 +143,8 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
         boolean allowDuplicateJMXDomains = config.getBoolean("allowDuplicateJMXDomains", true);
 
         if (clustered) {
-            gcb.transport().defaultTransport();
+            String nodeName = config.get("nodeName", System.getProperty(InfinispanConnectionProvider.JBOSS_NODE_NAME));
+            configureTransport(gcb, nodeName);
         }
         gcb.globalJmxStatistics().allowDuplicateDomains(allowDuplicateJMXDomains);
 
@@ -184,6 +189,13 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
         cacheManager.defineConfiguration(InfinispanConnectionProvider.AUTHORIZATION_CACHE_NAME, sessionCacheConfiguration);
         cacheManager.defineConfiguration(InfinispanConnectionProvider.AUTHENTICATION_SESSIONS_CACHE_NAME, sessionCacheConfiguration);
 
+        // Retrieve caches to enforce rebalance
+        cacheManager.getCache(InfinispanConnectionProvider.SESSION_CACHE_NAME, true);
+        cacheManager.getCache(InfinispanConnectionProvider.OFFLINE_SESSION_CACHE_NAME, true);
+        cacheManager.getCache(InfinispanConnectionProvider.LOGIN_FAILURE_CACHE_NAME, true);
+        cacheManager.getCache(InfinispanConnectionProvider.AUTHORIZATION_CACHE_NAME, true);
+        cacheManager.getCache(InfinispanConnectionProvider.AUTHENTICATION_SESSIONS_CACHE_NAME, true);
+
         ConfigurationBuilder replicationConfigBuilder = new ConfigurationBuilder();
         if (clustered) {
             replicationConfigBuilder.clustering().cacheMode(async ? CacheMode.REPL_ASYNC : CacheMode.REPL_SYNC);
@@ -278,14 +290,36 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
         ConfigurationBuilder cb = new ConfigurationBuilder();
 
         cb.eviction()
-          .strategy(EvictionStrategy.NONE)
-          .type(EvictionType.COUNT)
-          .size(InfinispanConnectionProvider.ACTION_TOKEN_CACHE_DEFAULT_MAX);
+                .strategy(EvictionStrategy.NONE)
+                .type(EvictionType.COUNT)
+                .size(InfinispanConnectionProvider.ACTION_TOKEN_CACHE_DEFAULT_MAX);
         cb.expiration()
-          .maxIdle(InfinispanConnectionProvider.ACTION_TOKEN_MAX_IDLE_SECONDS, TimeUnit.SECONDS)
-          .wakeUpInterval(InfinispanConnectionProvider.ACTION_TOKEN_WAKE_UP_INTERVAL_SECONDS, TimeUnit.SECONDS);
+                .maxIdle(InfinispanConnectionProvider.ACTION_TOKEN_MAX_IDLE_SECONDS, TimeUnit.SECONDS)
+                .wakeUpInterval(InfinispanConnectionProvider.ACTION_TOKEN_WAKE_UP_INTERVAL_SECONDS, TimeUnit.SECONDS);
 
         return cb.build();
     }
 
+    protected void configureTransport(GlobalConfigurationBuilder gcb, String nodeName) {
+        if (nodeName == null) {
+            gcb.transport().defaultTransport();
+        } else {
+            FileLookup fileLookup = FileLookupFactory.newInstance();
+
+            try {
+                // Compatibility with Wildfly
+                JChannel channel = new JChannel(fileLookup.lookupFileLocation("default-configs/default-jgroups-udp.xml", this.getClass().getClassLoader()));
+                channel.setName(nodeName);
+                JGroupsTransport transport = new JGroupsTransport(channel);
+
+                gcb.transport().nodeName(nodeName);
+                gcb.transport().transport(transport);
+
+                logger.infof("Configured jgroups transport with the channel name: %s", nodeName);
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
 }

model/infinispan/src/main/java/org/keycloak/connections/infinispan/InfinispanConnectionProvider.java

@@ -49,6 +49,9 @@ public interface InfinispanConnectionProvider extends Provider {
     int KEYS_CACHE_DEFAULT_MAX = 1000;
     int KEYS_CACHE_MAX_IDLE_SECONDS = 3600;
 
+    // System property used on Wildfly to identify distributedCache address and sticky session route
+    String JBOSS_NODE_NAME = "jboss.node.name";
+
 
     <K, V> Cache<K, V> getCache(String name);
 

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/AuthenticatedClientSessionAdapter.java

@@ -96,7 +96,7 @@ public class AuthenticatedClientSessionAdapter implements AuthenticatedClientSes
 
     @Override
     public String getId() {
-        return entity.getId();
+        return null;
     }
 
     @Override

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanStickySessionEncoderProvider.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.models.sessions.infinispan;
+
+import org.infinispan.Cache;
+import org.infinispan.distribution.DistributionManager;
+import org.infinispan.remoting.transport.Address;
+import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.sessions.StickySessionEncoderProvider;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class InfinispanStickySessionEncoderProvider implements StickySessionEncoderProvider {
+
+    private final KeycloakSession session;
+    private final String myNodeName;
+
+    public InfinispanStickySessionEncoderProvider(KeycloakSession session, String myNodeName) {
+        this.session = session;
+        this.myNodeName = myNodeName;
+    }
+
+    @Override
+    public String encodeSessionId(String sessionId) {
+        String nodeName = getNodeName(sessionId);
+        if (nodeName != null) {
+            return sessionId + '.' + nodeName;
+        } else {
+            return sessionId;
+        }
+    }
+
+    @Override
+    public String decodeSessionId(String encodedSessionId) {
+        int index = encodedSessionId.indexOf('.');
+        return index == -1 ? encodedSessionId : encodedSessionId.substring(0, index);
+    }
+
+    @Override
+    public void close() {
+
+    }
+
+
+    private String getNodeName(String sessionId) {
+        InfinispanConnectionProvider ispnProvider = session.getProvider(InfinispanConnectionProvider.class);
+        Cache cache = ispnProvider.getCache(InfinispanConnectionProvider.AUTHENTICATION_SESSIONS_CACHE_NAME);
+        DistributionManager distManager = cache.getAdvancedCache().getDistributionManager();
+
+        if (distManager != null) {
+            // Sticky session to the node, who owns this authenticationSession
+            Address address = distManager.getPrimaryLocation(sessionId);
+            return address.toString();
+        } else {
+            // Fallback to jbossNodeName if authSession cache is local
+            return myNodeName;
+        }
+    }
+
+
+}

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanStickySessionEncoderProviderFactory.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.models.sessions.infinispan;
+
+import org.keycloak.Config;
+import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.KeycloakSessionFactory;
+import org.keycloak.sessions.StickySessionEncoderProvider;
+import org.keycloak.sessions.StickySessionEncoderProviderFactory;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class InfinispanStickySessionEncoderProviderFactory implements StickySessionEncoderProviderFactory {
+
+    private String myNodeName;
+
+    @Override
+    public StickySessionEncoderProvider create(KeycloakSession session) {
+        return new InfinispanStickySessionEncoderProvider(session, myNodeName);
+    }
+
+    @Override
+    public void init(Config.Scope config) {
+        myNodeName = config.get("nodeName", System.getProperty(InfinispanConnectionProvider.JBOSS_NODE_NAME));
+    }
+
+    @Override
+    public void postInit(KeycloakSessionFactory factory) {
+
+    }
+
+    @Override
+    public void close() {
+
+    }
+
+    @Override
+    public String getId() {
+        return "infinispan";
+    }
+}

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/entities/AuthenticatedClientSessionEntity.java

@@ -26,7 +26,6 @@ import java.util.Set;
  */
 public class AuthenticatedClientSessionEntity implements Serializable {
 
-    private String id;
     private String authMethod;
     private String redirectUri;
     private int timestamp;
@@ -36,14 +35,6 @@ public class AuthenticatedClientSessionEntity implements Serializable {
     private Set<String> protocolMappers;
     private Map<String, String> notes;
 
-    public String getId() {
-        return id;
-    }
-
-    public void setId(String id) {
-        this.id = id;
-    }
-
     public String getAuthMethod() {
         return authMethod;
     }

model/infinispan/src/main/resources/META-INF/services/org.keycloak.sessions.StickySessionEncoderProviderFactory

@@ -0,0 +1,18 @@
+#
+# Copyright 2016 Red Hat, Inc. and/or its affiliates
+# and other contributors as indicated by the @author tags.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.keycloak.models.sessions.infinispan.InfinispanStickySessionEncoderProviderFactory

server-spi-private/src/main/java/org/keycloak/sessions/StickySessionEncoderProvider.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.sessions;
+
+import org.keycloak.provider.Provider;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public interface StickySessionEncoderProvider extends Provider {
+
+    String encodeSessionId(String sessionId);
+
+    String decodeSessionId(String encodedSessionId);
+
+}

server-spi-private/src/main/java/org/keycloak/sessions/StickySessionEncoderProviderFactory.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.sessions;
+
+import org.keycloak.provider.ProviderFactory;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public interface StickySessionEncoderProviderFactory extends ProviderFactory<StickySessionEncoderProvider> {
+}

server-spi-private/src/main/java/org/keycloak/sessions/StickySessionEncoderSpi.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.sessions;
+
+import org.keycloak.provider.Provider;
+import org.keycloak.provider.ProviderFactory;
+import org.keycloak.provider.Spi;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class StickySessionEncoderSpi implements Spi {
+
+    @Override
+    public boolean isInternal() {
+        return true;
+    }
+
+    @Override
+    public String getName() {
+        return "stickySessionEncoder";
+    }
+
+    @Override
+    public Class<? extends Provider> getProviderClass() {
+        return StickySessionEncoderProvider.class;
+    }
+
+    @Override
+    public Class<? extends ProviderFactory> getProviderFactoryClass() {
+        return StickySessionEncoderProviderFactory.class;
+    }
+}

server-spi-private/src/main/resources/META-INF/services/org.keycloak.provider.Spi

@@ -34,6 +34,7 @@ org.keycloak.scripting.ScriptingSpi
 org.keycloak.services.managers.BruteForceProtectorSpi
 org.keycloak.services.resource.RealmResourceSPI
 org.keycloak.sessions.AuthenticationSessionSpi
+org.keycloak.sessions.StickySessionEncoderSpi
 org.keycloak.protocol.ClientInstallationSpi
 org.keycloak.protocol.LoginProtocolSpi
 org.keycloak.protocol.ProtocolMapperSpi

services/src/main/java/org/keycloak/authentication/actiontoken/ActionTokenContext.java

@@ -115,7 +115,7 @@ public class ActionTokenContext<T extends JsonWebToken> {
         authSession = new AuthenticationSessionManager(session).createAuthenticationSession(realm, client, true);
         authSession.setAction(ClientSessionModel.Action.AUTHENTICATE.name());
         authSession.setProtocol(OIDCLoginProtocol.LOGIN_PROTOCOL);
-        String redirectUri = Urls.accountBase(uriInfo.getBaseUri()).path("/").build(realm.getName()).toString(); // TODO:mposolda It seems that this should be taken from client rather then hardcoded to account?
+        String redirectUri = Urls.accountBase(uriInfo.getBaseUri()).path("/").build(realm.getName()).toString();
         authSession.setRedirectUri(redirectUri);
         authSession.setClientNote(OIDCLoginProtocol.REDIRECT_URI_PARAM, redirectUri);
         authSession.setClientNote(OIDCLoginProtocol.RESPONSE_TYPE_PARAM, OAuth2Constants.CODE);

services/src/main/java/org/keycloak/authentication/actiontoken/idpverifyemail/IdpVerifyAccountLinkActionTokenHandler.java

@@ -86,6 +86,7 @@ public class IdpVerifyAccountLinkActionTokenHandler extends AbstractActionTokenH
 
             return tokenContext.getSession().getProvider(LoginFormsProvider.class)
                     .setSuccess(Messages.IDENTITY_PROVIDER_LINK_SUCCESS, token.getIdentityProviderAlias(), token.getIdentityProviderUsername())
+                    .setAttribute("skipLink", true)
                     .createInfoPage();
         }
 

services/src/main/java/org/keycloak/authentication/authenticators/broker/AbstractIdpAuthenticator.java

@@ -47,9 +47,6 @@ public abstract class AbstractIdpAuthenticator implements Authenticator {
     // The clientSession note flag to indicate that email provided by identityProvider was changed on updateProfile page
     public static final String UPDATE_PROFILE_EMAIL_CHANGED = "UPDATE_PROFILE_EMAIL_CHANGED";
 
-    // The clientSession note flag to indicate if re-authentication after first broker login happened in different browser window. This can happen for example during email verification
-    public static final String IS_DIFFERENT_BROWSER = "IS_DIFFERENT_BROWSER"; // TODO:mposolda can reuse the END_AFTER_REQUIRED_ACTIONS instead?
-
     // The clientSession note flag to indicate that updateProfile page will be always displayed even if "updateProfileOnFirstLogin" is off
     public static final String ENFORCE_UPDATE_PROFILE = "ENFORCE_UPDATE_PROFILE";
 

services/src/main/java/org/keycloak/authentication/authenticators/broker/IdpEmailVerificationAuthenticator.java

@@ -31,6 +31,7 @@ import org.keycloak.events.Errors;
 import org.keycloak.events.EventBuilder;
 import org.keycloak.events.EventType;
 import org.keycloak.forms.login.LoginFormsProvider;
+import org.keycloak.models.Constants;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.RealmModel;
 import org.keycloak.models.UserModel;
@@ -81,7 +82,13 @@ public class IdpEmailVerificationAuthenticator extends AbstractIdpAuthenticator
 
         UserModel existingUser = getExistingUser(session, realm, authSession);
 
-        sendVerifyEmail(session, context, existingUser, brokerContext);
+        // Do not allow resending e-mail by simple page refresh
+        if (! Objects.equals(authSession.getAuthNote(Constants.VERIFY_EMAIL_KEY), existingUser.getEmail())) {
+            authSession.setAuthNote(Constants.VERIFY_EMAIL_KEY, existingUser.getEmail());
+            sendVerifyEmail(session, context, existingUser, brokerContext);
+        } else {
+            showEmailSentPage(context, brokerContext);
+        }
     }
 
     @Override
@@ -89,7 +96,8 @@ public class IdpEmailVerificationAuthenticator extends AbstractIdpAuthenticator
         logger.debugf("Re-sending email requested for user, details follow");
 
         // This will allow user to re-send email again
-        context.getAuthenticationSession().removeAuthNote(VERIFY_ACCOUNT_IDP_USERNAME);
+        context.getAuthenticationSession().removeAuthNote(Constants.VERIFY_EMAIL_KEY);
+
         authenticateImpl(context, serializedCtx, brokerContext);
     }
 
@@ -146,6 +154,11 @@ public class IdpEmailVerificationAuthenticator extends AbstractIdpAuthenticator
             return;
         }
 
+        showEmailSentPage(context, brokerContext);
+    }
+
+
+    protected void showEmailSentPage(AuthenticationFlowContext context, BrokeredIdentityContext brokerContext) {
         String accessCode = context.generateAccessCode();
         URI action = context.getActionUrl(accessCode);
 
@@ -156,4 +169,5 @@ public class IdpEmailVerificationAuthenticator extends AbstractIdpAuthenticator
                 .createIdpLinkEmailPage();
         context.forceChallenge(challenge);
     }
+
 }

services/src/main/java/org/keycloak/protocol/oidc/TokenManager.java

@@ -197,31 +197,26 @@ public class TokenManager {
             return false;
         }
 
-        UserSessionModel userSession =  session.sessions().getUserSession(realm, token.getSessionState());
-        if (AuthenticationManager.isSessionValid(realm, userSession)) {
-            ClientSessionModel clientSession = session.sessions().getClientSession(realm, token.getClientSession());
-            if (clientSession != null) {
-                return true;
-            }
-        }
-
-<<<<<<< f392e79ad781014387c9fe5724815b24eab7a35f
-        userSession = session.sessions().getOfflineUserSession(realm, token.getSessionState());
-        if (AuthenticationManager.isOfflineSessionValid(realm, userSession)) {
-            ClientSessionModel clientSession = session.sessions().getOfflineClientSession(realm, token.getClientSession());
-            if (clientSession != null) {
-                return true;
-            }
-=======
         ClientModel client = realm.getClientByClientId(token.getIssuedFor());
         if (client == null || !client.isEnabled()) {
             return false;
         }
 
-        AuthenticatedClientSessionModel clientSession = userSession.getAuthenticatedClientSessions().get(client.getId());
-        if (clientSession == null) {
-            return false;
->>>>>>> KEYCLOAK-4626 AuthenticationSessions: start
+        UserSessionModel userSession =  session.sessions().getUserSession(realm, token.getSessionState());
+        if (AuthenticationManager.isSessionValid(realm, userSession)) {
+            AuthenticatedClientSessionModel clientSession = userSession.getAuthenticatedClientSessions().get(client.getId());
+            if (clientSession != null) {
+                return true;
+            }
+        }
+
+
+        userSession = session.sessions().getOfflineUserSession(realm, token.getSessionState());
+        if (AuthenticationManager.isOfflineSessionValid(realm, userSession)) {
+            AuthenticatedClientSessionModel clientSession = userSession.getAuthenticatedClientSessions().get(client.getId());
+            if (clientSession != null) {
+                return true;
+            }
         }
 
         return false;

services/src/main/java/org/keycloak/protocol/oidc/endpoints/UserInfoEndpoint.java

@@ -140,24 +140,7 @@ public class UserInfoEndpoint {
             throw new ErrorResponseException(OAuthErrorException.INVALID_TOKEN, "Token invalid: " + e.getMessage(), Response.Status.UNAUTHORIZED);
         }
 
-        UserSessionModel userSession = session.sessions().getUserSession(realm, token.getSessionState());
-        ClientSessionModel clientSession = session.sessions().getClientSession(token.getClientSession());
-        if( userSession == null ) {
-            userSession = session.sessions().getOfflineUserSession(realm, token.getSessionState());
-            if( AuthenticationManager.isOfflineSessionValid(realm, userSession)) {
-                clientSession = session.sessions().getOfflineClientSession(realm, token.getClientSession());
-            } else {
-                userSession = null;
-                clientSession = null;
-            }
-        }
-
-        if (userSession == null) {
-            event.error(Errors.USER_SESSION_NOT_FOUND);
-            throw new ErrorResponseException(OAuthErrorException.INVALID_REQUEST, "User session not found", Response.Status.BAD_REQUEST);
-        }
-
-        event.session(userSession);
+        UserSessionModel userSession = findValidSession(token, event);
 
         UserModel userModel = userSession.getUser();
         if (userModel == null) {
@@ -169,11 +152,6 @@ public class UserInfoEndpoint {
                 .detail(Details.USERNAME, userModel.getUsername());
 
 
-        if (clientSession == null || !AuthenticationManager.isSessionValid(realm, userSession)) {
-            event.error(Errors.SESSION_EXPIRED);
-            throw new ErrorResponseException(OAuthErrorException.INVALID_TOKEN, "Session expired", Response.Status.UNAUTHORIZED);
-        }
-
         ClientModel clientModel = realm.getClientByClientId(token.getIssuedFor());
         if (clientModel == null) {
             event.error(Errors.CLIENT_NOT_FOUND);
@@ -187,6 +165,12 @@ public class UserInfoEndpoint {
             throw new ErrorResponseException(OAuthErrorException.INVALID_REQUEST, "Client disabled", Response.Status.BAD_REQUEST);
         }
 
+        AuthenticatedClientSessionModel clientSession = userSession.getAuthenticatedClientSessions().get(clientModel.getId());
+        if (clientSession == null) {
+            event.error(Errors.SESSION_EXPIRED);
+            throw new ErrorResponseException(OAuthErrorException.INVALID_TOKEN, "Session expired", Response.Status.UNAUTHORIZED);
+        }
+
         AccessToken userInfo = new AccessToken();
         tokenManager.transformUserInfoAccessToken(session, userInfo, realm, clientModel, userModel, userSession, clientSession);
 
@@ -225,4 +209,34 @@ public class UserInfoEndpoint {
         return Cors.add(request, responseBuilder).auth().allowedOrigins(token).build();
     }
 
+
+    private UserSessionModel findValidSession(AccessToken token, EventBuilder event) {
+        UserSessionModel userSession = session.sessions().getUserSession(realm, token.getSessionState());
+        UserSessionModel offlineUserSession = null;
+        if (AuthenticationManager.isSessionValid(realm, userSession)) {
+            event.session(userSession);
+            return userSession;
+        } else {
+            offlineUserSession = session.sessions().getOfflineUserSession(realm, token.getSessionState());
+            if (AuthenticationManager.isOfflineSessionValid(realm, offlineUserSession)) {
+                event.session(offlineUserSession);
+                return offlineUserSession;
+            }
+        }
+
+        if (userSession == null && offlineUserSession == null) {
+            event.error(Errors.USER_SESSION_NOT_FOUND);
+            throw new ErrorResponseException(OAuthErrorException.INVALID_REQUEST, "User session not found", Response.Status.BAD_REQUEST);
+        }
+
+        if (userSession != null) {
+            event.session(userSession);
+        } else {
+            event.session(offlineUserSession);
+        }
+
+        event.error(Errors.SESSION_EXPIRED);
+        throw new ErrorResponseException(OAuthErrorException.INVALID_TOKEN, "Session expired", Response.Status.UNAUTHORIZED);
+    }
+
 }

services/src/main/java/org/keycloak/services/managers/AuthenticationManager.java

@@ -513,7 +513,7 @@ public class AuthenticationManager {
             
             if (actionTokenKey != null) {
                 ActionTokenStoreProvider actionTokenStore = session.getProvider(ActionTokenStoreProvider.class);
-                actionTokenStore.put(actionTokenKey, null);
+                actionTokenStore.put(actionTokenKey, null); // Token is invalidated
             }
         }
 

services/src/main/java/org/keycloak/services/managers/AuthenticationSessionManager.java

@@ -28,13 +28,14 @@ import org.keycloak.models.UserSessionModel;
 import org.keycloak.protocol.RestartLoginCookie;
 import org.keycloak.services.util.CookieHelper;
 import org.keycloak.sessions.AuthenticationSessionModel;
+import org.keycloak.sessions.StickySessionEncoderProvider;
 
 /**
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
 public class AuthenticationSessionManager {
 
-    private static final String AUTH_SESSION_ID = "AUTH_SESSION_ID";
+    public static final String AUTH_SESSION_ID = "AUTH_SESSION_ID";
 
     private static final Logger log = Logger.getLogger(AuthenticationSessionManager.class);
 
@@ -57,12 +58,12 @@ public class AuthenticationSessionManager {
 
 
     public String getCurrentAuthenticationSessionId(RealmModel realm) {
-        return getAuthSessionCookie();
+        return getAuthSessionCookieDecoded(realm);
     }
 
 
     public AuthenticationSessionModel getCurrentAuthenticationSession(RealmModel realm) {
-        String authSessionId = getAuthSessionCookie();
+        String authSessionId = getAuthSessionCookieDecoded(realm);
         return authSessionId==null ? null : session.authenticationSessions().getAuthenticationSession(realm, authSessionId);
     }
 
@@ -72,22 +73,37 @@ public class AuthenticationSessionManager {
         String cookiePath = AuthenticationManager.getRealmCookiePath(realm, uriInfo);
 
         boolean sslRequired = realm.getSslRequired().isRequired(session.getContext().getConnection());
-        CookieHelper.addCookie(AUTH_SESSION_ID, authSessionId, cookiePath, null, null, -1, sslRequired, true);
 
-        log.debugf("Set AUTH_SESSION_ID cookie with value %s", authSessionId);
+        StickySessionEncoderProvider encoder = session.getProvider(StickySessionEncoderProvider.class);
+        String encodedAuthSessionId = encoder.encodeSessionId(authSessionId);
+
+        CookieHelper.addCookie(AUTH_SESSION_ID, encodedAuthSessionId, cookiePath, null, null, -1, sslRequired, true);
+
+        log.debugf("Set AUTH_SESSION_ID cookie with value %s", encodedAuthSessionId);
     }
 
 
-    public String getAuthSessionCookie() {
+    private String getAuthSessionCookieDecoded(RealmModel realm) {
         String cookieVal = CookieHelper.getCookieValue(AUTH_SESSION_ID);
 
         if (cookieVal != null) {
             log.debugf("Found AUTH_SESSION_ID cookie with value %s", cookieVal);
+
+            StickySessionEncoderProvider encoder = session.getProvider(StickySessionEncoderProvider.class);
+            String decodedAuthSessionId = encoder.decodeSessionId(cookieVal);
+
+            // Check if owner of this authentication session changed due to re-hashing (usually node failover or addition of new node)
+            String reencoded = encoder.encodeSessionId(decodedAuthSessionId);
+            if (!reencoded.equals(cookieVal)) {
+                log.debugf("Route changed. Will update authentication session cookie");
+                setAuthSessionCookie(decodedAuthSessionId, realm);
+            }
+
+            return decodedAuthSessionId;
         } else {
             log.debugf("Not found AUTH_SESSION_ID cookie");
+            return null;
         }
-
-        return cookieVal;
     }
 
 

services/src/main/java/org/keycloak/services/resources/IdentityBrokerService.java

@@ -41,7 +41,6 @@ import org.keycloak.events.Details;
 import org.keycloak.events.Errors;
 import org.keycloak.events.EventBuilder;
 import org.keycloak.events.EventType;
-import org.keycloak.forms.login.LoginFormsProvider;
 import org.keycloak.models.AccountRoles;
 import org.keycloak.models.AuthenticatedClientSessionModel;
 import org.keycloak.models.AuthenticationFlowModel;
@@ -751,17 +750,7 @@ public class IdentityBrokerService implements IdentityProvider.AuthenticationCal
         UserModel federatedUser = authSession.getAuthenticatedUser();
 
         if (wasFirstBrokerLogin) {
-
-            String isDifferentBrowser = authSession.getAuthNote(AbstractIdpAuthenticator.IS_DIFFERENT_BROWSER);
-            if (Boolean.parseBoolean(isDifferentBrowser)) {
-                new AuthenticationSessionManager(session).removeAuthenticationSession(realmModel, authSession, true);
-                return session.getProvider(LoginFormsProvider.class)
-                        .setSuccess(Messages.IDENTITY_PROVIDER_LINK_SUCCESS, context.getIdpConfig().getAlias(), context.getUsername())
-                        .createInfoPage();
-            } else {
-                return finishBrokerAuthentication(context, federatedUser, authSession, providerId);
-            }
-
+            return finishBrokerAuthentication(context, federatedUser, authSession, providerId);
         } else {
 
             boolean firstBrokerLoginInProgress = (authSession.getAuthNote(AbstractIdpAuthenticator.BROKERED_CONTEXT_NOTE) != null);

services/src/main/java/org/keycloak/services/resources/LoginActionsService.java

@@ -338,7 +338,7 @@ public class LoginActionsService {
                 return ErrorPage.error(session, Messages.RESET_CREDENTIAL_NOT_ALLOWED);
 
             }
-            authSession = createAuthenticationSessionForClient(Constants.ACCOUNT_MANAGEMENT_CLIENT_ID);
+            authSession = createAuthenticationSessionForClient();
             return processResetCredentials(false, null, authSession);
         }
 
@@ -346,17 +346,17 @@ public class LoginActionsService {
         return resetCredentials(code, execution);
     }
 
-    AuthenticationSessionModel createAuthenticationSessionForClient(String clientId)
+    AuthenticationSessionModel createAuthenticationSessionForClient()
       throws UriBuilderException, IllegalArgumentException {
         AuthenticationSessionModel authSession;
 
         // set up the account service as the endpoint to call.
-        ClientModel client = realm.getClientByClientId(clientId == null ? Constants.ACCOUNT_MANAGEMENT_CLIENT_ID : clientId);
+        ClientModel client = realm.getClientByClientId(Constants.ACCOUNT_MANAGEMENT_CLIENT_ID);
         authSession = new AuthenticationSessionManager(session).createAuthenticationSession(realm, client, true);
         authSession.setAction(ClientSessionModel.Action.AUTHENTICATE.name());
         //authSession.setNote(AuthenticationManager.END_AFTER_REQUIRED_ACTIONS, "true");
         authSession.setProtocol(OIDCLoginProtocol.LOGIN_PROTOCOL);
-        String redirectUri = Urls.accountBase(uriInfo.getBaseUri()).path("/").build(realm.getName()).toString(); // TODO:mposolda It seems that this should be taken from client rather then hardcoded to account?
+        String redirectUri = Urls.accountBase(uriInfo.getBaseUri()).path("/").build(realm.getName()).toString();
         authSession.setRedirectUri(redirectUri);
         authSession.setClientNote(OIDCLoginProtocol.RESPONSE_TYPE_PARAM, OAuth2Constants.CODE);
         authSession.setClientNote(OIDCLoginProtocol.REDIRECT_URI_PARAM, redirectUri);

services/src/main/java/org/keycloak/services/util/BrowserHistoryHelper.java

@@ -72,20 +72,19 @@ public abstract class BrowserHistoryHelper {
             }
 
             // For now, handle just status 200 with String body. See if more is needed...
-            if (response.getStatus() == 200) {
-                Object entity = response.getEntity();
-                if (entity instanceof String) {
-                    String responseString = (String) entity;
+            Object entity = response.getEntity();
+            if (entity != null && entity instanceof String) {
+                String responseString = (String) entity;
 
-                    URI lastExecutionURL = new AuthenticationFlowURLHelper(session, session.getContext().getRealm(), session.getContext().getUri()).getLastExecutionUrl(authSession);
+                URI lastExecutionURL = new AuthenticationFlowURLHelper(session, session.getContext().getRealm(), session.getContext().getUri()).getLastExecutionUrl(authSession);
 
-                    // Inject javascript for history "replaceState"
-                    String responseWithJavascript = responseWithJavascript(responseString, lastExecutionURL.toString());
+                // Inject javascript for history "replaceState"
+                String responseWithJavascript = responseWithJavascript(responseString, lastExecutionURL.toString());
 
-                    return Response.fromResponse(response).entity(responseWithJavascript).build();
-                }
+                return Response.fromResponse(response).entity(responseWithJavascript).build();
             }
 
+
             return response;
         }
 

testsuite/integration-arquillian/HOW-TO-RUN.md

@@ -257,20 +257,20 @@ mvn -f testsuite/integration-arquillian/tests/other/console/pom.xml \
 ## Welcome Page tests
 The Welcome Page tests need to be run on WildFly/EAP and with `-Dskip.add.user.json` switch. So that they are disabled by default and are meant to be run separately.
 
-```
-# Prepare servers
-mvn -f testsuite/integration-arquillian/servers/pom.xml \
-    clean install \
-    -Pauth-server-wildfly \
-    -Papp-server-wildfly
 
-# Run tests
-mvn -f testsuite/integration-arquillian/tests/base/pom.xml \
-    clean test \
-    -Dtest=WelcomePageTest \
-    -Dskip.add.user.json \
-    -Pauth-server-wildfly
-```
+    # Prepare servers
+    mvn -f testsuite/integration-arquillian/servers/pom.xml \
+        clean install \
+        -Pauth-server-wildfly \
+        -Papp-server-wildfly
+
+    # Run tests
+    mvn -f testsuite/integration-arquillian/tests/base/pom.xml \
+        clean test \
+        -Dtest=WelcomePageTest \
+        -Dskip.add.user.json \
+        -Pauth-server-wildfly
+
 
 ## Social Login
 The social login tests require setup of all social networks including an example social user. These details can't be 
@@ -341,4 +341,80 @@ To run the X.509 client certificate authentication tests:
 	  -Dauth.server.ssl.required \
 	  -Dbrowser=phantomjs \
 	  "-Dtest=*.x509.*"
+	  
+## Cluster tests
+
+Cluster tests use 2 backend servers (Keycloak on Wildfly/EAP) and 1 frontend loadbalancer server node. Invalidation tests don't use loadbalancer. 
+The browser usually communicates directly with the backend node1 and after doing some change here (eg. updating user), it verifies that the change is visible on node2 and user is updated here as well.
+
+Failover tests use loadbalancer and they require the setup with the distributed infinispan caches switched to have 2 owners (default value is 1 owner). Otherwise failover won't reliably work. 
+
+
+The setup includes:
+
+*  a `mod_cluster` load balancer on Wildfly
+*  two clustered nodes of Keycloak server on Wildfly/EAP
+
+Clustering tests require MULTICAST to be enabled on machine's `loopback` network interface.
+This can be done by running the following commands under root privileges:
+
+    route add -net 224.0.0.0 netmask 240.0.0.0 dev lo
+    ifconfig lo multicast
+
+Then after build the sources, distribution and setup of clean shared database (replace command according your DB), you can use this command to setup servers:
+
+    export DB_HOST=localhost
+    mvn -f testsuite/integration-arquillian/servers/pom.xml \
+    -Pauth-server-wildfly,auth-server-cluster,jpa \
+    -Dsession.cache.owners=2 \
+    -Djdbc.mvn.groupId=mysql \
+    -Djdbc.mvn.version=5.1.29 \
+    -Djdbc.mvn.artifactId=mysql-connector-java \
+    -Dkeycloak.connectionsJpa.url=jdbc:mysql://$DB_HOST/keycloak \
+    -Dkeycloak.connectionsJpa.user=keycloak \
+    -Dkeycloak.connectionsJpa.password=keycloak \
+    clean install
+    
+And then this to run the cluster tests:
+   
+    mvn -f testsuite/integration-arquillian/tests/base/pom.xml \
+    -Pauth-server-wildfly,auth-server-cluster \
+    -Dsession.cache.owners=2 \
+    -Dbackends.console.output=true \
+    -Dauth.server.log.check=false \
+    -Dfrontend.console.output=true \
+    -Dtest=org.keycloak.testsuite.cluster.**.*Test clean install
+   
+	  
+### Cluster tests with embedded undertow
+
+#### Run cluster tests from IDE
+
+The test uses Undertow loadbalancer on `http://localhost:8180` and two embedded backend Undertow servers with Keycloak on `http://localhost:8181` and `http://localhost:8182` .
+You can use any cluster test (eg. AuthenticationSessionFailoverClusterTest) and run from IDE with those system properties (replace with your DB settings):
+
+    -Dauth.server.undertow=false -Dauth.server.undertow.cluster=true -Dauth.server.cluster=true 
+    -Dkeycloak.connectionsJpa.url=jdbc:mysql://localhost/keycloak -Dkeycloak.connectionsJpa.driver=com.mysql.jdbc.Driver 
+    -Dkeycloak.connectionsJpa.user=keycloak -Dkeycloak.connectionsJpa.password=keycloak -Dkeycloak.connectionsInfinispan.clustered=true -Dresources	 
+    -Dkeycloak.connectionsInfinispan.sessionsOwners=2 -Dsession.cache.owners=2    
+     
+Invalidation tests (subclass of `AbstractInvalidationClusterTest`) don't need last two properties.
+
+
+#### Run cluster environment from IDE
+
+This mode is useful for develop/manual tests of clustering features. You will need to manually run keycloak backend nodes and loadbalancer. 
+
+1) Run KeycloakServer server1 with:
+
+    -Dkeycloak.connectionsJpa.url=jdbc:mysql://localhost/keycloak -Dkeycloak.connectionsJpa.driver=com.mysql.jdbc.Driver 
+    -Dkeycloak.connectionsJpa.user=keycloak -Dkeycloak.connectionsJpa.password=keycloak -Dkeycloak.connectionsInfinispan.clustered=true 
+    -Dkeycloak.connectionsInfinispan.sessionsOwners=2 -Dresources
+
+and argument: `-p 8181`
+
+2) Run KeycloakServer server2 with same parameters but argument: `-p 8182`
+
+3) Run loadbalancer (class `SimpleUndertowLoadBalancer`) without arguments and system properties. Loadbalancer runs on port 8180, so you can access Keycloak on `http://localhost:8180/auth`     
+
 

testsuite/integration-arquillian/servers/auth-server/jboss/common/ispn-cache-owners.xsl

@@ -18,6 +18,11 @@
             <xsl:value-of select="$sessionCacheOwners"/>
         </xsl:attribute>
     </xsl:template>
+    <xsl:template match="//i:cache-container/i:distributed-cache[@name='authenticationSessions']/@owners">
+        <xsl:attribute name="owners">
+            <xsl:value-of select="$sessionCacheOwners"/>
+        </xsl:attribute>
+    </xsl:template>
     <xsl:template match="//i:cache-container/i:distributed-cache[@name='offlineSessions']/@owners">
         <xsl:attribute name="owners">
             <xsl:value-of select="$offlineSessionCacheOwners"/>

testsuite/integration-arquillian/servers/auth-server/undertow/src/main/java/org/keycloak/testsuite/arquillian/undertow/KeycloakOnUndertow.java

@@ -41,8 +41,11 @@ import org.jboss.shrinkwrap.api.spec.WebArchive;
 import org.jboss.shrinkwrap.descriptor.api.Descriptor;
 import org.jboss.shrinkwrap.undertow.api.UndertowWebArchive;
 import org.keycloak.common.util.reflections.Reflections;
+import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
+import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
 import org.keycloak.services.filters.KeycloakSessionServletFilter;
+import org.keycloak.services.managers.ApplianceBootstrap;
 import org.keycloak.services.resources.KeycloakApplication;
 
 import javax.servlet.DispatcherType;
@@ -106,6 +109,11 @@ public class KeycloakOnUndertow implements DeployableContainer<KeycloakOnUnderto
 
     @Override
     public ProtocolMetaData deploy(Archive<?> archive) throws DeploymentException {
+        if (isRemoteMode()) {
+            log.infof("Skipped deployment of '%s' as we are in remote mode!", archive.getName());
+            return new ProtocolMetaData();
+        }
+
         DeploymentInfo di = getDeplotymentInfoFromArchive(archive);
 
         ClassLoader parentCl = Thread.currentThread().getContextClassLoader();
@@ -152,7 +160,7 @@ public class KeycloakOnUndertow implements DeployableContainer<KeycloakOnUnderto
             return;
         }
 
-        log.info("Starting auth server on embedded Undertow.");
+        log.infof("Starting auth server on embedded Undertow on: http://%s:%d", configuration.getBindAddress(), configuration.getBindHttpPort());
         long start = System.currentTimeMillis();
 
         if (undertow == null) {
@@ -164,13 +172,37 @@ public class KeycloakOnUndertow implements DeployableContainer<KeycloakOnUnderto
                         .setIoThreads(configuration.getWorkerThreads() / 8)
         );
 
-        DeploymentInfo di = createAuthServerDeploymentInfo();
-        undertow.deploy(di);
-        ResteasyDeployment deployment = (ResteasyDeployment) di.getServletContextAttributes().get(ResteasyDeployment.class.getName());
-        sessionFactory = ((KeycloakApplication) deployment.getApplication()).getSessionFactory();
+        if (configuration.getRoute() != null) {
+            log.info("Using route: " + configuration.getRoute());
+        }
+
+        SetSystemProperty setRouteProperty = new SetSystemProperty(InfinispanConnectionProvider.JBOSS_NODE_NAME, configuration.getRoute());
+        try {
+            DeploymentInfo di = createAuthServerDeploymentInfo();
+            undertow.deploy(di);
+            ResteasyDeployment deployment = (ResteasyDeployment) di.getServletContextAttributes().get(ResteasyDeployment.class.getName());
+            sessionFactory = ((KeycloakApplication) deployment.getApplication()).getSessionFactory();
+
+            setupDevConfig();
+
+            log.info("Auth server started in " + (System.currentTimeMillis() - start) + " ms\n");
+        } finally {
+            setRouteProperty.revert();
+        }
+    }
 
 
-        log.info("Auth server started in " + (System.currentTimeMillis() - start) + " ms\n");
+    protected void setupDevConfig() {
+        KeycloakSession session = sessionFactory.create();
+        try {
+            session.getTransactionManager().begin();
+            if (new ApplianceBootstrap(session).isNoMasterUser()) {
+                new ApplianceBootstrap(session).createMasterRealmUser("admin", "admin");
+            }
+            session.getTransactionManager().commit();
+        } finally {
+            session.close();
+        }
     }
 
     @Override
@@ -187,11 +219,16 @@ public class KeycloakOnUndertow implements DeployableContainer<KeycloakOnUnderto
 
     private boolean isRemoteMode() {
         //return true;
-        return "true".equals(System.getProperty("remote.mode"));
+        return configuration.isRemoteMode();
     }
 
     @Override
     public void undeploy(Archive<?> archive) throws DeploymentException {
+        if (isRemoteMode()) {
+            log.infof("Skipped undeployment of '%s' as we are in remote mode!", archive.getName());
+            return;
+        }
+
         Field containerField = Reflections.findDeclaredField(UndertowJaxrsServer.class, "container");
         Reflections.setAccessible(containerField);
         ServletContainer container = (ServletContainer) Reflections.getFieldValue(containerField, undertow);

testsuite/integration-arquillian/servers/auth-server/undertow/src/main/java/org/keycloak/testsuite/arquillian/undertow/KeycloakOnUndertowArquillianExtension.java

@@ -2,6 +2,7 @@ package org.keycloak.testsuite.arquillian.undertow;
 
 import org.jboss.arquillian.container.spi.client.container.DeployableContainer;
 import org.jboss.arquillian.core.spi.LoadableExtension;
+import org.keycloak.testsuite.arquillian.undertow.lb.SimpleUndertowLoadBalancerContainer;
 
 /**
  *
@@ -12,6 +13,7 @@ public class KeycloakOnUndertowArquillianExtension implements LoadableExtension
     @Override
     public void register(ExtensionBuilder builder) {
         builder.service(DeployableContainer.class, KeycloakOnUndertow.class);
+        builder.service(DeployableContainer.class, SimpleUndertowLoadBalancerContainer.class);
     }
 
 }

testsuite/integration-arquillian/servers/auth-server/undertow/src/main/java/org/keycloak/testsuite/arquillian/undertow/KeycloakOnUndertowConfiguration.java

@@ -19,11 +19,18 @@ package org.keycloak.testsuite.arquillian.undertow;
 
 import org.arquillian.undertow.UndertowContainerConfiguration;
 import org.jboss.arquillian.container.spi.ConfigurationException;
+import org.jboss.logging.Logger;
 
 public class KeycloakOnUndertowConfiguration extends UndertowContainerConfiguration {
 
+    protected static final Logger log = Logger.getLogger(KeycloakOnUndertowConfiguration.class);
+
     private int workerThreads = Math.max(Runtime.getRuntime().availableProcessors(), 2) * 8;
     private String resourcesHome;
+    private boolean remoteMode;
+    private String route;
+
+    private int bindHttpPortOffset = 0;
 
     public int getWorkerThreads() {
         return workerThreads;
@@ -41,10 +48,39 @@ public class KeycloakOnUndertowConfiguration extends UndertowContainerConfigurat
         this.resourcesHome = resourcesHome;
     }
 
+    public int getBindHttpPortOffset() {
+        return bindHttpPortOffset;
+    }
+
+    public void setBindHttpPortOffset(int bindHttpPortOffset) {
+        this.bindHttpPortOffset = bindHttpPortOffset;
+    }
+
+    public String getRoute() {
+        return route;
+    }
+
+    public void setRoute(String route) {
+        this.route = route;
+    }
+
+    public boolean isRemoteMode() {
+        return remoteMode;
+    }
+
+    public void setRemoteMode(boolean remoteMode) {
+        this.remoteMode = remoteMode;
+    }
+
     @Override
     public void validate() throws ConfigurationException {
         super.validate();
-        
+
+        int basePort = getBindHttpPort();
+        int newPort = basePort + bindHttpPortOffset;
+        setBindHttpPort(newPort);
+        log.info("KeycloakOnUndertow will listen on port: " + newPort);
+
         // TODO validate workerThreads
         
     }

testsuite/integration-arquillian/servers/auth-server/undertow/src/main/java/org/keycloak/testsuite/arquillian/undertow/SetSystemProperty.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.testsuite.arquillian.undertow;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+class SetSystemProperty {
+
+    private String name;
+    private String oldValue;
+
+    public SetSystemProperty(String name, String value) {
+        this.name = name;
+        this.oldValue = System.getProperty(name);
+
+        if (value == null) {
+            if (oldValue != null) {
+                System.getProperties().remove(name);
+            }
+        } else {
+            System.setProperty(name, value);
+        }
+    }
+
+    public void revert() {
+        String value = System.getProperty(name);
+
+        if (oldValue == null) {
+            if (value != null) {
+                System.getProperties().remove(name);
+            }
+        } else {
+            System.setProperty(name, oldValue);
+        }
+    }
+
+}

testsuite/integration-arquillian/servers/auth-server/undertow/src/main/java/org/keycloak/testsuite/arquillian/undertow/lb/SimpleUndertowLoadBalancer.java

@@ -0,0 +1,298 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.testsuite.arquillian.undertow.lb;
+
+import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import io.undertow.Undertow;
+import io.undertow.server.HttpHandler;
+import io.undertow.server.HttpServerExchange;
+import io.undertow.server.handlers.ResponseCodeHandler;
+import io.undertow.server.handlers.proxy.ExclusivityChecker;
+import io.undertow.server.handlers.proxy.LoadBalancingProxyClient;
+import io.undertow.server.handlers.proxy.ProxyCallback;
+import io.undertow.server.handlers.proxy.ProxyClient;
+import io.undertow.server.handlers.proxy.ProxyConnection;
+import io.undertow.server.handlers.proxy.ProxyHandler;
+import io.undertow.util.AttachmentKey;
+import io.undertow.util.Headers;
+import org.jboss.logging.Logger;
+import org.keycloak.services.managers.AuthenticationSessionManager;
+
+/**
+ * Loadbalancer on embedded undertow. Supports sticky session over "AUTH_SESSION_ID" cookie and failover to different node when sticky node not available.
+ * Status 503 is returned just if all backend nodes are unavailable.
+ *
+ * To configure backend nodes, you can use system property like : -Dkeycloak.nodes="node1=http://localhost:8181,node2=http://localhost:8182"
+ *
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class SimpleUndertowLoadBalancer {
+
+    private static final Logger log = Logger.getLogger(SimpleUndertowLoadBalancer.class);
+
+    static final String DEFAULT_NODES = "node1=http://localhost:8181,node2=http://localhost:8182";
+
+    private final String host;
+    private final int port;
+    private final String nodesString;
+    private Undertow undertow;
+
+
+    public static void main(String[] args) throws Exception {
+        String nodes = System.getProperty("keycloak.nodes", DEFAULT_NODES);
+
+        SimpleUndertowLoadBalancer lb = new SimpleUndertowLoadBalancer("localhost", 8180, nodes);
+        lb.start();
+
+        Runtime.getRuntime().addShutdownHook(new Thread() {
+
+            @Override
+            public void run() {
+                lb.stop();
+            }
+
+        });
+    }
+
+
+    public SimpleUndertowLoadBalancer(String host, int port, String nodesString) {
+        this.host = host;
+        this.port = port;
+        this.nodesString = nodesString;
+        log.infof("Keycloak nodes: %s", nodesString);
+    }
+
+
+    public void start() {
+        Map<String, String> nodes = parseNodes(nodesString);
+        try {
+            HttpHandler proxyHandler = createHandler(nodes);
+
+            undertow = Undertow.builder()
+                    .addHttpListener(port, host)
+                    .setHandler(proxyHandler)
+                    .build();
+            undertow.start();
+
+            log.infof("Loadbalancer started and ready to serve requests on http://%s:%d", host, port);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+
+    public void stop() {
+        undertow.stop();
+    }
+
+
+    static Map<String, String> parseNodes(String nodes) {
+        String[] nodesArray = nodes.split(",");
+        Map<String, String> result = new HashMap<>();
+
+        for (String nodeStr : nodesArray) {
+            String[] node = nodeStr.trim().split("=");
+            if (node.length != 2) {
+                throw new IllegalArgumentException("Illegal node format in the configuration: " + nodeStr);
+            }
+            result.put(node[0].trim(), node[1].trim());
+        }
+
+        return result;
+    }
+
+
+    private HttpHandler createHandler(Map<String, String> backendNodes) throws Exception {
+
+        // TODO: configurable options if needed
+        String sessionCookieNames = AuthenticationSessionManager.AUTH_SESSION_ID;
+        int connectionsPerThread = 20;
+        int problemServerRetry = 5; // In case of unavailable node, we will try to ping him every 5 seconds to check if it's back
+        int maxTime = 3600000; // 1 hour for proxy request timeout, so we can debug the backend keycloak servers
+        int requestQueueSize = 10;
+        int cachedConnectionsPerThread = 10;
+        int connectionIdleTimeout = 60;
+        int maxRetryAttempts = backendNodes.size() - 1;
+
+        final LoadBalancingProxyClient lb = new CustomLoadBalancingClient(new ExclusivityChecker() {
+
+            @Override
+            public boolean isExclusivityRequired(HttpServerExchange exchange) {
+                //we always create a new connection for upgrade requests
+                return exchange.getRequestHeaders().contains(Headers.UPGRADE);
+            }
+
+        }, maxRetryAttempts)
+                .setConnectionsPerThread(connectionsPerThread)
+                .setMaxQueueSize(requestQueueSize)
+                .setSoftMaxConnectionsPerThread(cachedConnectionsPerThread)
+                .setTtl(connectionIdleTimeout)
+                .setProblemServerRetry(problemServerRetry);
+        String[] sessionIds = sessionCookieNames.split(",");
+        for (String id : sessionIds) {
+            lb.addSessionCookieName(id);
+        }
+
+        for (Map.Entry<String, String> node : backendNodes.entrySet()) {
+            String route = node.getKey();
+            URI uri = new URI(node.getValue());
+
+            lb.addHost(uri, route);
+            log.infof("Added host: %s, route: %s", uri.toString(), route);
+        }
+
+        ProxyHandler handler = new ProxyHandler(lb, maxTime, ResponseCodeHandler.HANDLE_404);
+        return handler;
+    }
+
+
+    private class CustomLoadBalancingClient extends LoadBalancingProxyClient {
+
+        private final int maxRetryAttempts;
+
+        public CustomLoadBalancingClient(ExclusivityChecker checker, int maxRetryAttempts) {
+            super(checker);
+            this.maxRetryAttempts = maxRetryAttempts;
+        }
+
+
+        @Override
+        protected Host selectHost(HttpServerExchange exchange) {
+            Host host = super.selectHost(exchange);
+            log.debugf("Selected host: %s, host available: %b", host.getUri().toString(), host.isAvailable());
+            exchange.putAttachment(SELECTED_HOST, host);
+            return host;
+        }
+
+
+        @Override
+        protected Host findStickyHost(HttpServerExchange exchange) {
+            Host stickyHost = super.findStickyHost(exchange);
+
+            if (stickyHost != null) {
+
+                if (!stickyHost.isAvailable()) {
+                    log.infof("Sticky host %s not available. Trying different hosts", stickyHost.getUri());
+                    return null;
+                } else {
+                    log.infof("Sticky host %s found and looks available", stickyHost.getUri());
+                }
+            }
+
+            return stickyHost;
+        }
+
+
+        @Override
+        public void getConnection(ProxyTarget target, HttpServerExchange exchange, ProxyCallback<ProxyConnection> callback, long timeout, TimeUnit timeUnit) {
+            long timeoutMs = timeUnit.toMillis(timeout);
+
+            ProxyCallbackDelegate callbackDelegate = new ProxyCallbackDelegate(this, callback, timeoutMs, maxRetryAttempts);
+            super.getConnection(target, exchange, callbackDelegate, timeout, timeUnit);
+        }
+
+    }
+
+
+    private static final AttachmentKey<LoadBalancingProxyClient.Host> SELECTED_HOST = AttachmentKey.create(LoadBalancingProxyClient.Host.class);
+    private static final AttachmentKey<Integer> REMAINING_RETRY_ATTEMPTS = AttachmentKey.create(Integer.class);
+
+
+    private class ProxyCallbackDelegate implements ProxyCallback<ProxyConnection> {
+
+        private final ProxyClient proxyClient;
+        private final ProxyCallback<ProxyConnection> delegate;
+        private final long timeoutMs;
+        private final int maxRetryAttempts;
+
+
+        public ProxyCallbackDelegate(ProxyClient proxyClient, ProxyCallback<ProxyConnection> delegate, long timeoutMs, int maxRetryAttempts) {
+            this.proxyClient = proxyClient;
+            this.delegate = delegate;
+            this.timeoutMs = timeoutMs;
+            this.maxRetryAttempts = maxRetryAttempts;
+        }
+
+
+        @Override
+        public void completed(HttpServerExchange exchange, ProxyConnection result) {
+            LoadBalancingProxyClient.Host host = exchange.getAttachment(SELECTED_HOST);
+            if (host == null) {
+                // shouldn't happen
+                log.error("Host is null!!!");
+            } else {
+                // Host was restored
+                if (!host.isAvailable()) {
+                    log.infof("Host %s available again", host.getUri());
+                    host.clearError();
+                }
+            }
+
+            delegate.completed(exchange, result);
+        }
+
+
+        @Override
+        public void failed(HttpServerExchange exchange) {
+            final long time = System.currentTimeMillis();
+
+            Integer remainingAttempts = exchange.getAttachment(REMAINING_RETRY_ATTEMPTS);
+            if (remainingAttempts == null) {
+                remainingAttempts = maxRetryAttempts;
+            } else {
+                remainingAttempts--;
+            }
+
+            exchange.putAttachment(REMAINING_RETRY_ATTEMPTS, remainingAttempts);
+
+            log.infof("Failed request to selected host. Remaining attempts: %d", remainingAttempts);
+            if (remainingAttempts > 0) {
+                if (timeoutMs > 0 && time > timeoutMs) {
+                    delegate.failed(exchange);
+                } else {
+                    ProxyClient.ProxyTarget target = proxyClient.findTarget(exchange);
+                    if (target != null) {
+                        final long remaining = timeoutMs > 0 ? timeoutMs - time : -1;
+                        proxyClient.getConnection(target, exchange, this, remaining, TimeUnit.MILLISECONDS);
+                    } else {
+                        couldNotResolveBackend(exchange); // The context was registered when we started, so return 503
+                    }
+                }
+            } else {
+                couldNotResolveBackend(exchange);
+            }
+        }
+
+
+        @Override
+        public void couldNotResolveBackend(HttpServerExchange exchange) {
+            delegate.couldNotResolveBackend(exchange);
+        }
+
+
+        @Override
+        public void queuedRequestFailed(HttpServerExchange exchange) {
+            delegate.queuedRequestFailed(exchange);
+        }
+
+    }
+}

testsuite/integration-arquillian/servers/auth-server/undertow/src/main/java/org/keycloak/testsuite/arquillian/undertow/lb/SimpleUndertowLoadBalancerConfiguration.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.testsuite.arquillian.undertow.lb;
+
+import org.arquillian.undertow.UndertowContainerConfiguration;
+import org.jboss.arquillian.container.spi.ConfigurationException;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class SimpleUndertowLoadBalancerConfiguration extends UndertowContainerConfiguration {
+
+    private String nodes = SimpleUndertowLoadBalancer.DEFAULT_NODES;
+
+    public String getNodes() {
+        return nodes;
+    }
+
+    public void setNodes(String nodes) {
+        this.nodes = nodes;
+    }
+
+    @Override
+    public void validate() throws ConfigurationException {
+        super.validate();
+
+        try {
+            SimpleUndertowLoadBalancer.parseNodes(nodes);
+        } catch (Exception e) {
+            throw new ConfigurationException(e);
+        }
+    }
+}

testsuite/integration-arquillian/servers/auth-server/undertow/src/main/java/org/keycloak/testsuite/arquillian/undertow/lb/SimpleUndertowLoadBalancerContainer.java

@@ -0,0 +1,87 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.testsuite.arquillian.undertow.lb;
+
+import org.jboss.arquillian.container.spi.client.container.DeployableContainer;
+import org.jboss.arquillian.container.spi.client.container.DeploymentException;
+import org.jboss.arquillian.container.spi.client.container.LifecycleException;
+import org.jboss.arquillian.container.spi.client.protocol.ProtocolDescription;
+import org.jboss.arquillian.container.spi.client.protocol.metadata.ProtocolMetaData;
+import org.jboss.logging.Logger;
+import org.jboss.shrinkwrap.api.Archive;
+import org.jboss.shrinkwrap.descriptor.api.Descriptor;
+
+/**
+ * Arquillian container over {@link SimpleUndertowLoadBalancer}
+ *
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class SimpleUndertowLoadBalancerContainer implements DeployableContainer<SimpleUndertowLoadBalancerConfiguration> {
+
+    private static final Logger log = Logger.getLogger(SimpleUndertowLoadBalancerContainer.class);
+
+    private SimpleUndertowLoadBalancerConfiguration configuration;
+    private SimpleUndertowLoadBalancer container;
+
+    @Override
+    public Class<SimpleUndertowLoadBalancerConfiguration> getConfigurationClass() {
+        return SimpleUndertowLoadBalancerConfiguration.class;
+    }
+
+    @Override
+    public void setup(SimpleUndertowLoadBalancerConfiguration configuration) {
+        this.configuration = configuration;
+    }
+
+    @Override
+    public void start() throws LifecycleException {
+        this.container = new SimpleUndertowLoadBalancer(configuration.getBindAddress(), configuration.getBindHttpPort(), configuration.getNodes());
+        this.container.start();
+    }
+
+    @Override
+    public void stop() throws LifecycleException {
+        log.info("Going to stop loadbalancer");
+        this.container.stop();
+    }
+
+    @Override
+    public ProtocolDescription getDefaultProtocol() {
+        return new ProtocolDescription("Servlet 3.1");
+    }
+
+    @Override
+    public ProtocolMetaData deploy(Archive<?> archive) throws DeploymentException {
+        throw new UnsupportedOperationException("Not implemented");
+    }
+
+    @Override
+    public void undeploy(Archive<?> archive) throws DeploymentException {
+        throw new UnsupportedOperationException("Not implemented");
+    }
+
+    @Override
+    public void deploy(Descriptor descriptor) throws DeploymentException {
+        throw new UnsupportedOperationException("Not implemented");
+    }
+
+    @Override
+    public void undeploy(Descriptor descriptor) throws DeploymentException {
+        throw new UnsupportedOperationException("Not implemented");
+    }
+}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/AuthServerTestEnricher.java

@@ -71,6 +71,8 @@ public class AuthServerTestEnricher {
     private static final String AUTH_SERVER_CLUSTER_PROPERTY = "auth.server.cluster";
     public static final boolean AUTH_SERVER_CLUSTER = Boolean.parseBoolean(System.getProperty(AUTH_SERVER_CLUSTER_PROPERTY, "false"));
 
+    private static final boolean AUTH_SERVER_UNDERTOW_CLUSTER = Boolean.parseBoolean(System.getProperty("auth.server.undertow.cluster", "false"));
+
     private static final Boolean START_MIGRATION_CONTAINER = "auto".equals(System.getProperty("migration.mode")) || 
             "manual".equals(System.getProperty("migration.mode"));
 
@@ -112,9 +114,25 @@ public class AuthServerTestEnricher {
 
         suiteContext = new SuiteContext(containers);
 
-        String authServerFrontend = AUTH_SERVER_CLUSTER
-                ? "auth-server-balancer-wildfly" // if cluster mode enabled, load-balancer is the frontend
-                : AUTH_SERVER_CONTAINER; // single-node mode
+        String authServerFrontend = null;
+
+        if (AUTH_SERVER_CLUSTER) {
+            // if cluster mode enabled, load-balancer is the frontend
+            for (ContainerInfo c : containers) {
+                if (c.getQualifier().startsWith("auth-server-balancer")) {
+                    authServerFrontend = c.getQualifier();
+                }
+            }
+
+            if (authServerFrontend != null) {
+                log.info("Using frontend container: " + authServerFrontend);
+            } else {
+                throw new IllegalStateException("Not found frontend container");
+            }
+        } else {
+            authServerFrontend = AUTH_SERVER_CONTAINER; // single-node mode
+        }
+
         String authServerBackend = AUTH_SERVER_CONTAINER + "-backend";
         int backends = 0;
         for (ContainerInfo container : suiteContext.getContainers()) {
@@ -130,6 +148,11 @@ public class AuthServerTestEnricher {
             }
         }
 
+        // Setup with 2 undertow backend nodes and no loadbalancer.
+//        if (AUTH_SERVER_UNDERTOW_CLUSTER && suiteContext.getAuthServerInfo() == null && !suiteContext.getAuthServerBackendsInfo().isEmpty()) {
+//            suiteContext.setAuthServerInfo(suiteContext.getAuthServerBackendsInfo().get(0));
+//        }
+
         // validate auth server setup
         if (suiteContext.getAuthServerInfo() == null) {
             throw new RuntimeException(String.format("No auth server container matching '%s' found in arquillian.xml.", authServerFrontend));

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/AdminClientUtil.java

@@ -33,6 +33,7 @@ import org.jboss.resteasy.plugins.providers.jackson.ResteasyJackson2Provider;
 import org.keycloak.admin.client.Keycloak;
 import org.keycloak.models.Constants;
 import org.keycloak.testsuite.arquillian.AuthServerTestEnricher;
+import org.keycloak.testsuite.arquillian.SuiteContext;
 
 import static org.keycloak.testsuite.auth.page.AuthRealm.ADMIN;
 import static org.keycloak.testsuite.auth.page.AuthRealm.MASTER;
@@ -41,7 +42,7 @@ import static org.keycloak.testsuite.util.IOUtil.PROJECT_BUILD_DIRECTORY;
 
 public class AdminClientUtil {
 
-    public static Keycloak createAdminClient(boolean ignoreUnknownProperties) throws Exception {
+    public static Keycloak createAdminClient(boolean ignoreUnknownProperties, String authServerContextRoot) throws Exception {
         SSLContext ssl = null;
         if ("true".equals(System.getProperty("auth.server.ssl.required"))) {
             File trustore = new File(PROJECT_BUILD_DIRECTORY, "dependency/keystore/keycloak.truststore");
@@ -61,12 +62,12 @@ public class AdminClientUtil {
             jacksonProvider.setMapper(objectMapper);
         }
 
-        return Keycloak.getInstance(AuthServerTestEnricher.getAuthServerContextRoot() + "/auth",
+        return Keycloak.getInstance(authServerContextRoot + "/auth",
                 MASTER, ADMIN, ADMIN, Constants.ADMIN_CLI_CLIENT_ID, null, ssl, jacksonProvider);
     }
 
     public static Keycloak createAdminClient() throws Exception {
-        return createAdminClient(false);
+        return createAdminClient(false, AuthServerTestEnricher.getAuthServerContextRoot());
     }
 
     private static SSLContext getSSLContextWithTrustore(File file, String password) throws CertificateException, NoSuchAlgorithmException, KeyStoreException, IOException, KeyManagementException {

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/AbstractKeycloakTest.java

@@ -18,26 +18,19 @@ package org.keycloak.testsuite;
 
 import org.apache.commons.configuration.ConfigurationException;
 import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.http.ssl.SSLContexts;
 import org.keycloak.common.util.KeycloakUriBuilder;
 import org.keycloak.common.util.Time;
 import org.keycloak.testsuite.arquillian.KcArquillian;
 import org.keycloak.testsuite.arquillian.TestContext;
 
-import java.io.File;
-import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.security.KeyManagementException;
-import java.security.KeyStoreException;
-import java.security.NoSuchAlgorithmException;
-import java.security.cert.CertificateException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import javax.net.ssl.SSLContext;
+
 import javax.ws.rs.NotFoundException;
 import org.jboss.arquillian.container.test.api.RunAsClient;
 import org.jboss.arquillian.drone.api.annotation.Drone;
@@ -53,7 +46,6 @@ import org.keycloak.admin.client.resource.RealmResource;
 import org.keycloak.admin.client.resource.RealmsResource;
 import org.keycloak.admin.client.resource.UserResource;
 import org.keycloak.admin.client.resource.UsersResource;
-import org.keycloak.models.Constants;
 import org.keycloak.representations.idm.RealmRepresentation;
 import org.keycloak.representations.idm.RequiredActionProviderRepresentation;
 import org.keycloak.representations.idm.UserRepresentation;
@@ -77,7 +69,6 @@ import org.openqa.selenium.WebDriver;
 import static org.keycloak.testsuite.admin.Users.setPasswordFor;
 import static org.keycloak.testsuite.auth.page.AuthRealm.ADMIN;
 import static org.keycloak.testsuite.auth.page.AuthRealm.MASTER;
-import static org.keycloak.testsuite.util.IOUtil.PROJECT_BUILD_DIRECTORY;
 
 /**
  *
@@ -135,7 +126,8 @@ public abstract class AbstractKeycloakTest {
     public void beforeAbstractKeycloakTest() throws Exception {
         adminClient = testContext.getAdminClient();
         if (adminClient == null) {
-            adminClient = AdminClientUtil.createAdminClient(suiteContext.isAdapterCompatTesting());
+            String authServerContextRoot = suiteContext.getAuthServerInfo().getContextRoot().toString();
+            adminClient = AdminClientUtil.createAdminClient(suiteContext.isAdapterCompatTesting(), authServerContextRoot);
             testContext.setAdminClient(adminClient);
         }
 
@@ -147,10 +139,9 @@ public abstract class AbstractKeycloakTest {
 
         TestEventsLogger.setDriver(driver);
 
-        if (!suiteContext.isAdminPasswordUpdated()) {
-            log.debug("updating admin password");
+        // The backend cluster nodes may not be yet started. Password will be updated later for cluster setup.
+        if (!AuthServerTestEnricher.AUTH_SERVER_CLUSTER) {
             updateMasterAdminPassword();
-            suiteContext.setAdminPasswordUpdated(true);
         }
 
         if (testContext.getTestRealmReps() == null) {
@@ -202,10 +193,16 @@ public abstract class AbstractKeycloakTest {
         return false;
     }
 
-    private void updateMasterAdminPassword() {
-        welcomePage.navigateTo();
-        if (!welcomePage.isPasswordSet()) {
-            welcomePage.setPassword("admin", "admin");
+    protected void updateMasterAdminPassword() {
+        if (!suiteContext.isAdminPasswordUpdated()) {
+            log.debug("updating admin password");
+
+            welcomePage.navigateTo();
+            if (!welcomePage.isPasswordSet()) {
+                welcomePage.setPassword("admin", "admin");
+            }
+
+            suiteContext.setAdminPasswordUpdated(true);
         }
     }
 
@@ -236,7 +233,8 @@ public abstract class AbstractKeycloakTest {
         if (testingClient == null) {
             testingClient = testContext.getTestingClient();
             if (testingClient == null) {
-                testingClient = KeycloakTestingClient.getInstance(AuthServerTestEnricher.getAuthServerContextRoot() + "/auth");
+                String authServerContextRoot = suiteContext.getAuthServerInfo().getContextRoot().toString();
+                testingClient = KeycloakTestingClient.getInstance(authServerContextRoot + "/auth");
                 testContext.setTestingClient(testingClient);
             }
         }

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/adapter/example/authorization/AbstractServletAuthzAdapterTest.java

@@ -28,6 +28,8 @@ import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.List;
 
+import javax.ws.rs.core.Response;
+
 import org.jboss.arquillian.container.test.api.Deployer;
 import org.jboss.arquillian.test.api.ArquillianResource;
 import org.junit.BeforeClass;
@@ -199,7 +201,8 @@ public abstract class AbstractServletAuthzAdapterTest extends AbstractExampleAda
 
         assertFalse(policy.getUsers().isEmpty());
 
-        getAuthorizationResource().policies().user().create(policy);
+        Response response = getAuthorizationResource().policies().user().create(policy);
+        response.close();
     }
 
     protected interface ExceptionRunnable {

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/adapter/example/authorization/AbstractServletAuthzFunctionalAdapterTest.java

@@ -23,6 +23,8 @@ import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 
+import javax.ws.rs.core.Response;
+
 import org.jboss.arquillian.container.test.api.Deployment;
 import org.jboss.shrinkwrap.api.spec.WebArchive;
 import org.junit.Test;
@@ -289,7 +291,8 @@ public abstract class AbstractServletAuthzFunctionalAdapterTest extends Abstract
             policy.addClient("admin-cli");
 
             ClientPoliciesResource policyResource = getAuthorizationResource().policies().client();
-            policyResource.create(policy);
+            Response response = policyResource.create(policy);
+            response.close();
             policy = policyResource.findByName(policy.getName());
 
             updatePermissionPolicies("Protected Resource Permission", policy.getName());

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/admin/PermissionsTest.java

@@ -46,8 +46,6 @@ import org.keycloak.representations.idm.RealmRepresentation;
 import org.keycloak.representations.idm.RequiredActionProviderRepresentation;
 import org.keycloak.representations.idm.RequiredActionProviderSimpleRepresentation;
 import org.keycloak.representations.idm.RoleRepresentation;
-import org.keycloak.representations.idm.UserFederationMapperRepresentation;
-import org.keycloak.representations.idm.UserFederationProviderRepresentation;
 import org.keycloak.representations.idm.UserRepresentation;
 import org.keycloak.representations.idm.authorization.PolicyRepresentation;
 import org.keycloak.representations.idm.authorization.ResourceRepresentation;
@@ -64,7 +62,6 @@ import org.keycloak.testsuite.util.FederatedIdentityBuilder;
 import org.keycloak.testsuite.util.GreenMailRule;
 import org.keycloak.testsuite.util.IdentityProviderBuilder;
 import org.keycloak.testsuite.util.RealmBuilder;
-import org.keycloak.testsuite.util.TestCleanup;
 import org.keycloak.testsuite.util.UserBuilder;
 
 import javax.ws.rs.ClientErrorException;

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/admin/client/authorization/AbstractPolicyManagementTest.java

@@ -28,6 +28,8 @@ import java.util.List;
 import java.util.Set;
 import java.util.function.Supplier;
 
+import javax.ws.rs.core.Response;
+
 import org.junit.Before;
 import org.keycloak.admin.client.resource.ClientResource;
 import org.keycloak.admin.client.resource.ClientsResource;
@@ -39,7 +41,6 @@ import org.keycloak.representations.idm.authorization.ResourceRepresentation;
 import org.keycloak.representations.idm.authorization.ScopeRepresentation;
 import org.keycloak.representations.idm.authorization.UserPolicyRepresentation;
 import org.keycloak.testsuite.AbstractKeycloakTest;
-import org.keycloak.testsuite.util.AdminClientUtil;
 import org.keycloak.testsuite.util.ClientBuilder;
 import org.keycloak.testsuite.util.RealmBuilder;
 import org.keycloak.testsuite.util.UserBuilder;
@@ -131,7 +132,10 @@ public abstract class AbstractPolicyManagementTest extends AbstractKeycloakTest
         resources.add(new ResourceRepresentation("Resource B", scopes));
         resources.add(new ResourceRepresentation("Resource C", scopes));
 
-        resources.forEach(resource -> getClient().authorization().resources().create(resource));
+        resources.forEach(resource -> {
+            Response response = getClient().authorization().resources().create(resource);
+            response.close();
+        });
     }
 
     private void createPolicies(RealmResource realm, ClientResource client) throws IOException {
@@ -147,7 +151,8 @@ public abstract class AbstractPolicyManagementTest extends AbstractKeycloakTest
         representation.setName(name);
         representation.addUser(userId);
 
-        client.authorization().policies().user().create(representation);
+        Response response = client.authorization().policies().user().create(representation);
+        response.close();
     }
 
     protected ClientResource getClient() {
@@ -161,7 +166,7 @@ public abstract class AbstractPolicyManagementTest extends AbstractKeycloakTest
 
     protected RealmResource getRealm() {
         try {
-            return AdminClientUtil.createAdminClient().realm("authz-test");
+            return adminClient.realm("authz-test");
         } catch (Exception cause) {
             throw new RuntimeException("Failed to create admin client", cause);
         }

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/admin/client/authorization/ClientPolicyManagementTest.java

@@ -112,6 +112,7 @@ public class ClientPolicyManagementTest extends AbstractPolicyManagementTest {
         ClientPoliciesResource policies = authorization.policies().client();
         Response response = policies.create(representation);
         ClientPolicyRepresentation created = response.readEntity(ClientPolicyRepresentation.class);
+        response.close();
 
         policies.findById(created.getId()).remove();
 
@@ -136,6 +137,7 @@ public class ClientPolicyManagementTest extends AbstractPolicyManagementTest {
         ClientPoliciesResource policies = authorization.policies().client();
         Response response = policies.create(representation);
         ClientPolicyRepresentation created = response.readEntity(ClientPolicyRepresentation.class);
+        response.close();
 
         PolicyResource policy = authorization.policies().policy(created.getId());
         PolicyRepresentation genericConfig = policy.toRepresentation();
@@ -152,6 +154,7 @@ public class ClientPolicyManagementTest extends AbstractPolicyManagementTest {
         ClientPoliciesResource permissions = authorization.policies().client();
         Response response = permissions.create(representation);
         ClientPolicyRepresentation created = response.readEntity(ClientPolicyRepresentation.class);
+        response.close();
         ClientPolicyResource permission = permissions.findById(created.getId());
         assertRepresentation(representation, permission);
     }

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/authz/ConflictingScopePermissionTest.java

@@ -50,7 +50,6 @@ import org.keycloak.representations.idm.authorization.PolicyRepresentation;
 import org.keycloak.representations.idm.authorization.ResourcePermissionRepresentation;
 import org.keycloak.representations.idm.authorization.ScopePermissionRepresentation;
 import org.keycloak.testsuite.AbstractKeycloakTest;
-import org.keycloak.testsuite.util.AdminClientUtil;
 import org.keycloak.testsuite.util.ClientBuilder;
 import org.keycloak.testsuite.util.RealmBuilder;
 import org.keycloak.testsuite.util.UserBuilder;
@@ -140,7 +139,7 @@ public class ConflictingScopePermissionTest extends AbstractKeycloakTest {
     }
 
     private RealmResource getRealm() throws Exception {
-        return AdminClientUtil.createAdminClient().realm("authz-test");
+        return adminClient.realm("authz-test");
     }
 
     private ClientResource getClient(RealmResource realm) {

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/authz/RequireUmaAuthorizationScopeTest.java

@@ -24,6 +24,8 @@ import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 
+import javax.ws.rs.core.Response;
+
 import org.junit.Before;
 import org.junit.Test;
 import org.keycloak.admin.client.resource.AuthorizationResource;
@@ -43,7 +45,6 @@ import org.keycloak.representations.idm.authorization.JSPolicyRepresentation;
 import org.keycloak.representations.idm.authorization.ResourcePermissionRepresentation;
 import org.keycloak.representations.idm.authorization.ResourceRepresentation;
 import org.keycloak.testsuite.AbstractKeycloakTest;
-import org.keycloak.testsuite.util.AdminClientUtil;
 import org.keycloak.testsuite.util.ClientBuilder;
 import org.keycloak.testsuite.util.RealmBuilder;
 import org.keycloak.testsuite.util.RoleBuilder;
@@ -77,14 +78,16 @@ public class RequireUmaAuthorizationScopeTest extends AbstractKeycloakTest {
         AuthorizationResource authorization = client.authorization();
         ResourceRepresentation resource = new ResourceRepresentation("Resource A");
 
-        authorization.resources().create(resource);
+        Response response = authorization.resources().create(resource);
+        response.close();
 
         JSPolicyRepresentation policy = new JSPolicyRepresentation();
 
         policy.setName("Default Policy");
         policy.setCode("$evaluation.grant();");
 
-        authorization.policies().js().create(policy);
+        response = authorization.policies().js().create(policy);
+        response.close();
 
         ResourcePermissionRepresentation permission = new ResourcePermissionRepresentation();
 
@@ -92,7 +95,8 @@ public class RequireUmaAuthorizationScopeTest extends AbstractKeycloakTest {
         permission.addResource(resource.getName());
         permission.addPolicy(policy.getName());
 
-        authorization.permissions().resource().create(permission);
+        response = authorization.permissions().resource().create(permission);
+        response.close();
     }
 
     @Test
@@ -140,7 +144,7 @@ public class RequireUmaAuthorizationScopeTest extends AbstractKeycloakTest {
     }
 
     private RealmResource getRealm() throws Exception {
-        return AdminClientUtil.createAdminClient().realm("authz-test");
+        return adminClient.realm("authz-test");
     }
 
     private ClientResource getClient(RealmResource realm) {

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractClusterTest.java

@@ -45,6 +45,12 @@ public abstract class AbstractClusterTest extends AbstractKeycloakTest {
         logFailoverSetup();
     }
 
+    // Assume that route like "node6" will have corresponding backend container like "auth-server-wildfly-backend6"
+    protected void setCurrentFailNodeForRoute(String route) {
+        String routeNumber = route.substring(route.length() - 1);
+        currentFailNodeIndex = Integer.parseInt(routeNumber) - 1;
+    }
+
     protected ContainerInfo getCurrentFailNode() {
         return backendNode(currentFailNodeIndex);
     }
@@ -111,9 +117,13 @@ public abstract class AbstractClusterTest extends AbstractKeycloakTest {
     }
 
     protected Keycloak getAdminClientFor(ContainerInfo node) {
-        return node.equals(suiteContext.getAuthServerInfo())
-                ? adminClient // frontend client
-                : backendAdminClients.get(node);
+        Keycloak adminClient = backendAdminClients.get(node);
+
+        if (adminClient == null && node.equals(suiteContext.getAuthServerInfo())) {
+            adminClient = this.adminClient;
+        }
+
+        return adminClient;
     }
 
     @Before

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractFailoverClusterTest.java

@@ -0,0 +1,112 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.testsuite.cluster;
+
+
+import java.util.List;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.keycloak.representations.idm.RealmRepresentation;
+import org.keycloak.testsuite.page.AbstractPage;
+import org.keycloak.testsuite.page.PageWithLogOutAction;
+import org.openqa.selenium.Cookie;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.keycloak.testsuite.auth.page.AuthRealm.ADMIN;
+import static org.keycloak.testsuite.util.URLAssert.assertCurrentUrlStartsWith;
+import static org.keycloak.testsuite.util.WaitUtils.pause;
+
+public abstract class AbstractFailoverClusterTest extends AbstractClusterTest {
+
+    public static final String KEYCLOAK_SESSION_COOKIE = "KEYCLOAK_SESSION";
+
+    public static final Integer SESSION_CACHE_OWNERS = Integer.parseInt(System.getProperty("session.cache.owners", "1"));
+    public static final Integer OFFLINE_SESSION_CACHE_OWNERS = Integer.parseInt(System.getProperty("offline.session.cache.owners", "1"));
+    public static final Integer LOGIN_FAILURES_CACHE_OWNERS = Integer.parseInt(System.getProperty("login.failure.cache.owners", "1"));
+
+    public static final Integer REBALANCE_WAIT = Integer.parseInt(System.getProperty("rebalance.wait", "5000"));
+
+    @Override
+    public void addTestRealms(List<RealmRepresentation> testRealms) {
+    }
+
+
+    /**
+     * failure --> failback --> failure of next node
+     */
+    protected void switchFailedNode() {
+        assertFalse(controller.isStarted(getCurrentFailNode().getQualifier()));
+
+        failback();
+        pause(REBALANCE_WAIT);
+
+        iterateCurrentFailNode();
+
+        failure();
+        pause(REBALANCE_WAIT);
+
+        assertFalse(controller.isStarted(getCurrentFailNode().getQualifier()));
+    }
+
+    protected Cookie login(AbstractPage targetPage) {
+        targetPage.navigateTo();
+        assertCurrentUrlStartsWith(loginPage);
+        loginPage.form().login(ADMIN, ADMIN);
+        assertCurrentUrlStartsWith(targetPage);
+        Cookie sessionCookie = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
+        assertNotNull(sessionCookie);
+        return sessionCookie;
+    }
+
+    protected void logout(AbstractPage targetPage) {
+        if (!(targetPage instanceof PageWithLogOutAction)) {
+            throw new IllegalArgumentException(targetPage.getClass().getSimpleName() + " must implement PageWithLogOutAction interface");
+        }
+        targetPage.navigateTo();
+        assertCurrentUrlStartsWith(targetPage);
+        ((PageWithLogOutAction) targetPage).logOut();
+    }
+
+    protected Cookie verifyLoggedIn(AbstractPage targetPage, Cookie sessionCookieForVerification) {
+        // verify on realm path
+        masterRealmPage.navigateTo();
+        Cookie sessionCookieOnRealmPath = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
+        assertNotNull(sessionCookieOnRealmPath);
+        assertEquals(sessionCookieOnRealmPath.getValue(), sessionCookieForVerification.getValue());
+        // verify on target page
+        targetPage.navigateTo();
+        assertCurrentUrlStartsWith(targetPage);
+        Cookie sessionCookie = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
+        assertNotNull(sessionCookie);
+        assertEquals(sessionCookie.getValue(), sessionCookieForVerification.getValue());
+        return sessionCookie;
+    }
+
+    protected void verifyLoggedOut(AbstractPage targetPage) {
+        // verify on target page
+        targetPage.navigateTo();
+        driver.navigate().refresh();
+        assertCurrentUrlStartsWith(loginPage);
+        Cookie sessionCookie = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
+        assertNull(sessionCookie);
+    }
+}

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AuthenticationSessionFailoverClusterTest.java

@@ -0,0 +1,171 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.testsuite.cluster;
+
+import java.io.IOException;
+import java.util.List;
+
+import javax.mail.MessagingException;
+
+import org.jboss.arquillian.graphene.page.Page;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.keycloak.models.UserModel;
+import org.keycloak.representations.idm.RealmRepresentation;
+import org.keycloak.representations.idm.UserRepresentation;
+import org.keycloak.services.managers.AuthenticationSessionManager;
+import org.keycloak.testsuite.AbstractKeycloakTest;
+import org.keycloak.testsuite.Assert;
+import org.keycloak.testsuite.admin.ApiUtil;
+import org.keycloak.testsuite.pages.AppPage;
+import org.keycloak.testsuite.pages.LoginPage;
+import org.keycloak.testsuite.pages.LoginPasswordUpdatePage;
+import org.keycloak.testsuite.pages.LoginUpdateProfilePage;
+import org.keycloak.testsuite.util.UserBuilder;
+import org.openqa.selenium.Cookie;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.keycloak.testsuite.admin.AbstractAdminTest.loadJson;
+import static org.keycloak.testsuite.util.WaitUtils.pause;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class AuthenticationSessionFailoverClusterTest extends AbstractFailoverClusterTest {
+
+    private String userId;
+
+    @Page
+    protected LoginPage loginPage;
+
+    @Page
+    protected LoginPasswordUpdatePage updatePasswordPage;
+
+
+    @Page
+    protected LoginUpdateProfilePage updateProfilePage;
+
+    @Page
+    protected AppPage appPage;
+
+
+    @Before
+    public void setup() {
+        try {
+            adminClient.realm("test").remove();
+        } catch (Exception ignore) {
+        }
+
+        RealmRepresentation testRealm = loadJson(getClass().getResourceAsStream("/testrealm.json"), RealmRepresentation.class);
+        adminClient.realms().create(testRealm);
+
+        UserRepresentation user = UserBuilder.create()
+                .username("login-test")
+                .email("login@test.com")
+                .enabled(true)
+                .requiredAction(UserModel.RequiredAction.UPDATE_PASSWORD.toString())
+                .requiredAction(UserModel.RequiredAction.UPDATE_PROFILE.toString())
+                .build();
+
+        userId = ApiUtil.createUserAndResetPasswordWithAdminClient(adminClient.realm("test"), user, "password");
+        getCleanup().addUserId(userId);
+
+        oauth.clientId("test-app");
+    }
+
+    @After
+    public void after() {
+        adminClient.realm("test").remove();
+    }
+
+
+    @Test
+    public void failoverDuringAuthentication() throws Exception {
+
+        boolean expectSuccessfulFailover = SESSION_CACHE_OWNERS >= 2;
+
+        log.info("AUTHENTICATION FAILOVER TEST: cluster size = " + getClusterSize() + ", session-cache owners = " + SESSION_CACHE_OWNERS
+                + " --> Testsing for " + (expectSuccessfulFailover ? "" : "UN") + "SUCCESSFUL session failover.");
+
+        assertEquals(2, getClusterSize());
+
+        failoverTest(expectSuccessfulFailover);
+    }
+
+
+    protected void failoverTest(boolean expectSuccessfulFailover) throws IOException, MessagingException {
+        loginPage.open();
+
+        String cookieValue1 = getAuthSessionCookieValue();
+
+        // Login and assert on "updatePassword" page
+        loginPage.login("login-test", "password");
+        updatePasswordPage.assertCurrent();
+
+        // Route didn't change
+        Assert.assertEquals(cookieValue1, getAuthSessionCookieValue());
+
+        log.info("Authentication session cookie: " + cookieValue1);
+
+        setCurrentFailNodeForRoute(cookieValue1);
+
+        failure();
+        pause(REBALANCE_WAIT);
+        logFailoverSetup();
+
+        // Trigger the action now
+        updatePasswordPage.changePassword("password", "password");
+
+        if (expectSuccessfulFailover) {
+            //Action was successful
+            updateProfilePage.assertCurrent();
+
+            String cookieValue2 = getAuthSessionCookieValue();
+
+            log.info("Authentication session cookie after failover: " + cookieValue2);
+
+            // Cookie was moved to the second node
+            Assert.assertEquals(cookieValue1.substring(0, 36), cookieValue2.substring(0, 36));
+            Assert.assertNotEquals(cookieValue1, cookieValue2);
+
+        } else {
+            loginPage.assertCurrent();
+            String error = loginPage.getError();
+            log.info("Failover not successful as expected. Error on login page: " + error);
+            Assert.assertNotNull(error);
+
+            loginPage.login("login-test", "password");
+            updatePasswordPage.changePassword("password", "password");
+        }
+
+
+        updateProfilePage.assertCurrent();
+
+        // Successfully update profile and assert user logged
+        updateProfilePage.update("John", "Doe3", "john@doe3.com");
+        appPage.assertCurrent();
+    }
+
+    private String getAuthSessionCookieValue() {
+        Cookie authSessionCookie = driver.manage().getCookieNamed(AuthenticationSessionManager.AUTH_SESSION_ID);
+        Assert.assertNotNull(authSessionCookie);
+        return authSessionCookie.getValue();
+    }
+}

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/SessionFailoverClusterTest.java

@@ -2,38 +2,16 @@ package org.keycloak.testsuite.cluster;
 
 import org.junit.Before;
 import org.junit.Test;
-import org.keycloak.representations.idm.RealmRepresentation;
-import org.keycloak.testsuite.page.AbstractPage;
-import org.keycloak.testsuite.page.PageWithLogOutAction;
 import org.openqa.selenium.Cookie;
 
-import java.util.List;
-
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.keycloak.testsuite.auth.page.AuthRealm.ADMIN;
-import static org.keycloak.testsuite.util.URLAssert.assertCurrentUrlStartsWith;
 import static org.keycloak.testsuite.util.WaitUtils.pause;
 
 /**
  *
  * @author tkyjovsk
  */
-public class SessionFailoverClusterTest extends AbstractClusterTest {
-
-    public static final String KEYCLOAK_SESSION_COOKIE = "KEYCLOAK_SESSION";
-
-    public static final Integer SESSION_CACHE_OWNERS = Integer.parseInt(System.getProperty("session.cache.owners", "1"));
-    public static final Integer OFFLINE_SESSION_CACHE_OWNERS = Integer.parseInt(System.getProperty("offline.session.cache.owners", "1"));
-    public static final Integer LOGIN_FAILURES_CACHE_OWNERS = Integer.parseInt(System.getProperty("login.failure.cache.owners", "1"));
-
-    public static final Integer REBALANCE_WAIT = Integer.parseInt(System.getProperty("rebalance.wait", "5000"));
-
-    @Override
-    public void addTestRealms(List<RealmRepresentation> testRealms) {
-    }
+public class SessionFailoverClusterTest extends AbstractFailoverClusterTest {
 
     @Before
     public void beforeSessionFailover() {
@@ -45,7 +23,7 @@ public class SessionFailoverClusterTest extends AbstractClusterTest {
     @Test
     public void sessionFailover() {
 
-        boolean expectSuccessfulFailover = SESSION_CACHE_OWNERS >= getClusterSize();
+        boolean expectSuccessfulFailover = SESSION_CACHE_OWNERS >= 2;
 
         log.info("SESSION FAILOVER TEST: cluster size = " + getClusterSize() + ", session-cache owners = " + SESSION_CACHE_OWNERS
                 + " --> Testsing for " + (expectSuccessfulFailover ? "" : "UN") + "SUCCESSFUL session failover.");
@@ -91,64 +69,4 @@ public class SessionFailoverClusterTest extends AbstractClusterTest {
 
     }
 
-    /**
-     * failure --> failback --> failure of next node
-     */
-    protected void switchFailedNode() {
-        assertFalse(controller.isStarted(getCurrentFailNode().getQualifier()));
-        
-        failback();
-        pause(REBALANCE_WAIT);
-        
-        iterateCurrentFailNode();
-        
-        failure();
-        pause(REBALANCE_WAIT);
-        
-        assertFalse(controller.isStarted(getCurrentFailNode().getQualifier()));
-    }
-
-    protected Cookie login(AbstractPage targetPage) {
-        targetPage.navigateTo();
-        assertCurrentUrlStartsWith(loginPage);
-        loginPage.form().login(ADMIN, ADMIN);
-        assertCurrentUrlStartsWith(targetPage);
-        Cookie sessionCookie = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
-        assertNotNull(sessionCookie);
-        return sessionCookie;
-    }
-
-    protected void logout(AbstractPage targetPage) {
-        if (!(targetPage instanceof PageWithLogOutAction)) {
-            throw new IllegalArgumentException(targetPage.getClass().getSimpleName() + " must implement PageWithLogOutAction interface");
-        }
-        targetPage.navigateTo();
-        assertCurrentUrlStartsWith(targetPage);
-        ((PageWithLogOutAction) targetPage).logOut();
-    }
-
-    protected Cookie verifyLoggedIn(AbstractPage targetPage, Cookie sessionCookieForVerification) {
-        // verify on realm path
-        masterRealmPage.navigateTo();
-        Cookie sessionCookieOnRealmPath = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
-        assertNotNull(sessionCookieOnRealmPath);
-        assertEquals(sessionCookieOnRealmPath.getValue(), sessionCookieForVerification.getValue());
-        // verify on target page
-        targetPage.navigateTo();
-        assertCurrentUrlStartsWith(targetPage);
-        Cookie sessionCookie = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
-        assertNotNull(sessionCookie);
-        assertEquals(sessionCookie.getValue(), sessionCookieForVerification.getValue());
-        return sessionCookie;
-    }
-
-    protected void verifyLoggedOut(AbstractPage targetPage) {
-        // verify on target page
-        targetPage.navigateTo();
-        driver.navigate().refresh();
-        assertCurrentUrlStartsWith(loginPage);
-        Cookie sessionCookie = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
-        assertNull(sessionCookie);
-    }
-
 }

testsuite/integration-arquillian/tests/base/src/test/resources/META-INF/keycloak-server.json

@@ -108,8 +108,9 @@
     "connectionsInfinispan": {
         "default": {
             "clustered": "${keycloak.connectionsInfinispan.clustered:false}",
-            "async": "${keycloak.connectionsInfinispan.async:true}",
-            "sessionsOwners": "${keycloak.connectionsInfinispan.sessionsOwners:2}",
+            "async": "${keycloak.connectionsInfinispan.async:false}",
+            "sessionsOwners": "${keycloak.connectionsInfinispan.sessionsOwners:1}",
+            "l1Lifespan": "${keycloak.connectionsInfinispan.l1Lifespan:600000}",
             "remoteStoreEnabled": "${keycloak.connectionsInfinispan.remoteStoreEnabled:false}",
             "remoteStoreHost": "${keycloak.connectionsInfinispan.remoteStoreHost:localhost}",
             "remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort:11222}"

testsuite/integration-arquillian/tests/base/src/test/resources/arquillian.xml

@@ -53,6 +53,7 @@
             <property name="bindAddress">localhost</property>
             <property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.KeycloakOnUndertow</property>
             <property name="bindHttpPort">${auth.server.http.port}</property>
+            <property name="remoteMode">${undertow.remote}</property>
         </configuration>
     </container>
     
@@ -90,7 +91,7 @@
                 <property name="jbossHome">${auth.server.backend1.home}</property>
                 <property name="serverConfig">standalone-ha.xml</property>
                 <property name="jbossArguments">
-                    -Djboss.socket.binding.port-offset=${auth.server.backend1.port.offset} 
+                    -Djboss.socket.binding.port-offset=${auth.server.backend1.port.offset}
                     -Djboss.node.name=node1
                     ${adapter.test.props}
                     ${auth.server.profile}
@@ -127,6 +128,43 @@
         </container>
     </group>
 
+    <!-- Clustering with embedded undertow -->
+    <group qualifier="auth-server-undertow-cluster">
+        <container qualifier="auth-server-undertow-backend1" mode="manual" >
+            <configuration>
+                <property name="enabled">${auth.server.undertow.cluster}</property>
+                <property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.KeycloakOnUndertow</property>
+                <property name="bindAddress">localhost</property>
+                <property name="bindHttpPort">${auth.server.http.port}</property>
+                <property name="bindHttpPortOffset">1</property>
+                <property name="route">node1</property>
+                <property name="remoteMode">${undertow.remote}</property>
+            </configuration>
+        </container>
+        <container qualifier="auth-server-undertow-backend2" mode="manual" >
+            <configuration>
+                <property name="enabled">${auth.server.undertow.cluster}</property>
+                <property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.KeycloakOnUndertow</property>
+                <property name="bindAddress">localhost</property>
+                <property name="bindHttpPort">${auth.server.http.port}</property>
+                <property name="bindHttpPortOffset">2</property>
+                <property name="route">node2</property>
+                <property name="remoteMode">${undertow.remote}</property>
+            </configuration>
+        </container>
+
+        <container qualifier="auth-server-balancer-undertow" mode="suite" >
+            <configuration>
+                <property name="enabled">${auth.server.undertow.cluster}</property>
+                <property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.lb.SimpleUndertowLoadBalancerContainer</property>
+                <property name="bindAddress">localhost</property>
+                <property name="bindHttpPort">${auth.server.http.port}</property>
+                <property name="nodes">node1=http://localhost:8181,node2=http://localhost:8182</property>
+            </configuration>
+        </container>
+    </group>
+
+
     <container qualifier="auth-server-balancer-wildfly" mode="suite" >
         <configuration>
             <property name="enabled">${auth.server.cluster}</property>

testsuite/integration/src/test/java/org/keycloak/testsuite/KeycloakServer.java

@@ -27,6 +27,7 @@ import org.jboss.logging.Logger;
 import org.jboss.resteasy.plugins.server.servlet.HttpServlet30Dispatcher;
 import org.jboss.resteasy.plugins.server.undertow.UndertowJaxrsServer;
 import org.jboss.resteasy.spi.ResteasyDeployment;
+import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
 import org.keycloak.models.RealmModel;
@@ -37,12 +38,15 @@ import org.keycloak.services.managers.RealmManager;
 import org.keycloak.services.resources.KeycloakApplication;
 import org.keycloak.testsuite.util.cli.TestsuiteCLI;
 import org.keycloak.util.JsonSerialization;
+import org.mvel2.util.Make;
 
 import javax.servlet.DispatcherType;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Properties;
 
 /**
@@ -187,6 +191,8 @@ public class KeycloakServer {
             config.setWorkerThreads(undertowWorkerThreads);
         }
 
+        detectNodeName(config);
+
         final KeycloakServer keycloak = new KeycloakServer(config);
         keycloak.sysout = true;
         keycloak.start();
@@ -369,4 +375,24 @@ public class KeycloakServer {
         return new File(s.toString());
     }
 
+
+    private static void detectNodeName(KeycloakServerConfig config) {
+        String nodeName = System.getProperty(InfinispanConnectionProvider.JBOSS_NODE_NAME);
+        if (nodeName == null) {
+            // Try to autodetect "jboss.node.name" from the port
+            Map<Integer, String> nodesCfg = new HashMap<>();
+            nodesCfg.put(8181, "node1");
+            nodesCfg.put(8182, "node2");
+
+            nodeName = nodesCfg.get(config.getPort());
+            if (nodeName != null) {
+                System.setProperty(InfinispanConnectionProvider.JBOSS_NODE_NAME, nodeName);
+            }
+        }
+
+        if (nodeName != null) {
+            log.infof("Node name: %s", nodeName);
+        }
+    }
+
 }

testsuite/integration/src/test/java/org/keycloak/testsuite/broker/AbstractFirstBrokerLoginTest.java

@@ -346,7 +346,7 @@ public abstract class AbstractFirstBrokerLoginTest extends AbstractIdentityProvi
         driver.navigate().to(linkFromMail.trim());
 
         infoPage.assertCurrent();
-        Assert.assertThat(infoPage.getInfo(), startsWith("Your account was successfully linked with " + getProviderId() + " account pedroigor"));
+        Assert.assertThat(infoPage.getInfo(), startsWith("You successfully verified your email. Please go back to your original browser and continue there with the login."));
     }
 
     /**
@@ -384,15 +384,13 @@ public abstract class AbstractFirstBrokerLoginTest extends AbstractIdentityProvi
 
             // authenticated, but not redirected to app. Just seeing info page.
             infoPage2.assertCurrent();
-            Assert.assertThat(infoPage2.getInfo(), startsWith("Your account was successfully linked with " + getProviderId() + " account pedroigor"));
+            Assert.assertThat(infoPage2.getInfo(), startsWith("You successfully verified your email. Please go back to your original browser and continue there with the login."));
         } finally {
             // Revert everything
             webRule2.after();
         }
 
-        driver.navigate().refresh();
-        this.loginExpiredPage.assertCurrent();
-        this.loginExpiredPage.clickLoginContinueLink();
+        this.idpLinkEmailPage.clickContinueFlowLink();
 
         // authenticated and redirected to app. User is linked with identity provider
         assertFederatedUser("pedroigor", "psilva@redhat.com", "pedroigor");

testsuite/integration/src/test/java/org/keycloak/testsuite/pages/IdpLinkEmailPage.java

@@ -31,6 +31,9 @@ public class IdpLinkEmailPage extends AbstractPage {
     @FindBy(linkText = "Click here")
     private WebElement resendEmailLink;
 
+    @FindBy(linkText = "Click here") // Actually same link like "resendEmailLink"
+    private WebElement continueFlowLink;
+
     @Override
     public boolean isCurrent() {
         return driver.getTitle().startsWith("Link ");
@@ -40,8 +43,8 @@ public class IdpLinkEmailPage extends AbstractPage {
         resendEmailLink.click();
     }
 
-    public String getResendEmailLink() {
-        return resendEmailLink.getAttribute("href");
+    public void clickContinueFlowLink() {
+        continueFlowLink.click();
     }
 
     @Override

testsuite/integration/src/test/resources/META-INF/keycloak-server.json

@@ -90,7 +90,7 @@
             "sessionsOwners": "${keycloak.connectionsInfinispan.sessionsOwners:1}",
             "l1Lifespan": "${keycloak.connectionsInfinispan.l1Lifespan:600000}",
             "remoteStoreEnabled": "${keycloak.connectionsInfinispan.remoteStoreEnabled:false}",
-            "remoteStoreHost": "${keycloak.connectionsInfinispan.remoteStoreHost:localhost}",
+            "remoteStoreHost": "${keycloak.connectionsjen neInfinispan.remoteStoreHost:localhost}",
             "remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort:11222}"
         }
     },

testsuite/integration/src/test/resources/log4j.properties

@@ -83,3 +83,5 @@ log4j.logger.org.apache.directory.server.ldap.LdapProtocolHandler=error
 log4j.logger.org.keycloak.authentication.authenticators.browser.IdentityProviderAuthenticator=trace
 log4j.logger.org.keycloak.services.resources.IdentityBrokerService=trace
 log4j.logger.org.keycloak.broker=trace
+
+# log4j.logger.io.undertow=trace

themes/src/main/resources/theme/base/login/login-idp-link-email.ftl

@@ -11,5 +11,8 @@
         <p id="instruction2" class="instruction">
             ${msg("emailLinkIdp2")} <a href="${url.loginAction}">${msg("doClickHere")}</a> ${msg("emailLinkIdp3")}
         </p>
+        <p id="instruction3" class="instruction">
+            ${msg("emailLinkIdp4")} <a href="${url.loginAction}">${msg("doClickHere")}</a> ${msg("emailLinkIdp5")}
+        </p>
     </#if>
 </@layout.registrationLayout>

themes/src/main/resources/theme/base/login/messages/messages_en.properties

@@ -83,6 +83,8 @@ emailLinkIdpTitle=Link {0}
 emailLinkIdp1=An email with instructions to link {0} account {1} with your {2} account has been sent to you.
 emailLinkIdp2=Haven''t received a verification code in your email?
 emailLinkIdp3=to re-send the email.
+emailLinkIdp4=If you already verified the email in different browser
+emailLinkIdp5=to continue.
 
 backToLogin=« Back to Login
 
@@ -208,7 +210,7 @@ sessionNotActiveMessage=Session not active.
 invalidCodeMessage=An error occurred, please login again through your application.
 identityProviderUnexpectedErrorMessage=Unexpected error when authenticating with identity provider
 identityProviderNotFoundMessage=Could not find an identity provider with the identifier.
-identityProviderLinkSuccess=Your account was successfully linked with {0} account {1} .
+identityProviderLinkSuccess=You successfully verified your email. Please go back to your original browser and continue there with the login.
 staleCodeMessage=This page is no longer valid, please go back to your application and login again
 realmSupportsNoCredentialsMessage=Realm does not support any credential type.
 identityProviderNotUniqueMessage=Realm supports multiple identity providers. Could not determine which identity provider should be used to authenticate with.