KEYCLOAK-4630 Added SessionsPreloadCrossDCTest for test preloading sessions and offline sessions. Support for manual.mode to control manually lifecycle of all servers.

This commit is contained in:
mposolda 2017-08-11 16:43:03 +02:00
parent 1289e84cdb
commit 868e76fcf3
8 changed files with 315 additions and 14 deletions

View file

@ -35,7 +35,7 @@ import java.util.concurrent.Future;
* Startup initialization for reading persistent userSessions to be filled into infinispan/memory . In cluster,
* the initialization is distributed among all cluster nodes, so the startup time is even faster
*
* TODO: Move to clusterService. Implementation is already pretty generic and doesn't contain any "userSession" specific stuff. All sessions-specific logic is in the SessionLoader implementation
* Implementation is pretty generic and doesn't contain any "userSession" specific stuff. All logic related to how are sessions loaded is in the SessionLoader implementation
*
* @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
*/

View file

@ -102,17 +102,14 @@ public class RemoteCacheSessionsLoader implements SessionLoader {
RemoteCache<?, ?> remoteCache = InfinispanUtil.getRemoteCache(cache);
// TODO:mposolda
log.infof("Will do bulk load of sessions from remote cache '%s' . First: %d, max: %d", cache.getName(), first, max);
log.debugf("Will do bulk load of sessions from remote cache '%s' . First: %d, max: %d", cache.getName(), first, max);
Map<String, Integer> remoteParams = new HashMap<>();
remoteParams.put("first", first);
remoteParams.put("max", max);
Map<byte[], byte[]> remoteObjects = remoteCache.execute("load-sessions.js", remoteParams);
// TODO:mposolda
log.infof("Finished loading sessions '%s' . First: %d, max: %d", cache.getName(), first, max);
log.debugf("Successfully finished loading sessions '%s' . First: %d, max: %d", cache.getName(), first, max);
Marshaller marshaller = remoteCache.getRemoteCacheManager().getMarshaller();

View file

@ -468,6 +468,16 @@ or
It can be useful to add additional system property to enable logging:
-Dkeycloak.infinispan.logging.level=debug
Tests from package "manual" uses manual lifecycle for all servers, so needs to be executed manually. Also needs to be executed with real DB like MySQL. You can run them with:
mvn -Pcache-server-infinispan -Dtest=*.crossdc.manual.* -Dmanual.mode=true \
-Dkeycloak.connectionsJpa.url.crossdc=jdbc:mysql://localhost/keycloak -Dkeycloak.connectionsJpa.driver.crossdc=com.mysql.jdbc.Driver \
-Dkeycloak.connectionsJpa.user=keycloak -Dkeycloak.connectionsJpa.password=keycloak \
-pl testsuite/integration-arquillian/tests/base test
@ -512,6 +522,9 @@ connects to the remoteStore provided by infinispan server configured in previous
-Dkeycloak.connectionsInfinispan.remoteStorePort=11222 -Dkeycloak.connectionsInfinispan.remoteStorePort.2=11222 -Dkeycloak.connectionsInfinispan.sessionsOwners=1
-Dsession.cache.owners=1 -Dkeycloak.infinispan.logging.level=debug -Dresources
NOTE: Tests from package "manual" (eg. SessionsPreloadCrossDCTest) needs to be executed with managed containers.
So skip steps 1,2 and add property `-Dmanual.mode=true` and change "cache.server.lifecycle.skip" to false `-Dcache.server.lifecycle.skip=false` or remove it.
7) If you want to debug and test manually, the servers are running on these ports (Note that not all backend servers are running by default and some might be also unused by loadbalancer):
Loadbalancer -> "http://localhost:8180/auth"

View file

@ -195,7 +195,13 @@ public class SimpleUndertowLoadBalancer {
@Override
protected Host selectHost(HttpServerExchange exchange) {
Host host = super.selectHost(exchange);
log.debugf("Selected host: %s, host available: %b", host.getUri().toString(), host.isAvailable());
if (host != null) {
log.debugf("Selected host: %s, host available: %b", host.getUri().toString(), host.isAvailable());
} else {
log.warn("No host available");
}
exchange.putAttachment(SELECTED_HOST, host);
return host;
}

View file

@ -54,7 +54,7 @@ import javax.ws.rs.NotFoundException;
*/
public class AuthServerTestEnricher {
protected final Logger log = Logger.getLogger(this.getClass());
protected static final Logger log = Logger.getLogger(AuthServerTestEnricher.class);
@Inject
private Instance<ContainerRegistry> containerRegistry;
@ -84,6 +84,10 @@ public class AuthServerTestEnricher {
private static final Boolean START_MIGRATION_CONTAINER = "auto".equals(System.getProperty("migration.mode")) ||
"manual".equals(System.getProperty("migration.mode"));
// In manual mode are all containers despite loadbalancers started in mode "manual" and nothing is managed through "suite".
// Useful for tests, which require restart servers etc.
private static final String MANUAL_MODE = "manual.mode";
@Inject
@SuiteScoped
private InstanceProducer<SuiteContext> suiteContextProducer;
@ -118,6 +122,9 @@ public class AuthServerTestEnricher {
.map(ContainerInfo::new)
.collect(Collectors.toSet());
// A way to specify that containers should be in mode "manual" rather then "suite"
checkManualMode(containers);
suiteContext = new SuiteContext(containers);
if (AUTH_SERVER_CROSS_DC) {
@ -148,6 +155,15 @@ public class AuthServerTestEnricher {
suiteContext.addAuthServerBackendsInfo(Integer.valueOf(dcString), c);
});
containers.stream()
.filter(c -> c.getQualifier().startsWith("cache-server-cross-dc-"))
.sorted((a, b) -> a.getQualifier().compareTo(b.getQualifier()))
.forEach(containerInfo -> {
int prefixSize = "cache-server-cross-dc-".length();
int dcIndex = Integer.parseInt(containerInfo.getQualifier().substring(prefixSize)) -1;
suiteContext.addCacheServerInfo(dcIndex, containerInfo);
});
if (suiteContext.getDcAuthServerInfo().isEmpty()) {
throw new RuntimeException(String.format("No auth server container matching '%s' found in arquillian.xml.", AUTH_SERVER_BACKEND));
}
@ -157,6 +173,9 @@ public class AuthServerTestEnricher {
if (suiteContext.getDcAuthServerBackendsInfo().stream().anyMatch(List::isEmpty)) {
throw new RuntimeException(String.format("Some data center has no auth server container matching '%s' defined in arquillian.xml.", AUTH_SERVER_BACKEND));
}
if (suiteContext.getCacheServersInfo().isEmpty()) {
throw new IllegalStateException("Cache containers misconfiguration");
}
log.info("Using frontend containers: " + this.suiteContext.getDcAuthServerInfo().stream()
.map(ContainerInfo::getQualifier)
@ -270,10 +289,23 @@ public class AuthServerTestEnricher {
public void afterClass(@Observes(precedence = 2) AfterClass event) {
TestContext testContext = testContextProducer.get();
List<RealmRepresentation> testRealmReps = testContext.getTestRealmReps();
Keycloak adminClient = testContext.getAdminClient();
KeycloakTestingClient testingClient = testContext.getTestingClient();
removeTestRealms(testContext, adminClient);
if (adminClient != null) {
adminClient.close();
}
if (testingClient != null) {
testingClient.close();
}
}
public static void removeTestRealms(TestContext testContext, Keycloak adminClient) {
List<RealmRepresentation> testRealmReps = testContext.getTestRealmReps();
if (testRealmReps != null) {
log.info("removing test realms after test class");
for (RealmRepresentation testRealm : testRealmReps) {
@ -286,13 +318,20 @@ public class AuthServerTestEnricher {
}
}
}
}
if (adminClient != null) {
adminClient.close();
}
if (testingClient != null) {
testingClient.close();
private void checkManualMode(Set<ContainerInfo> containers) {
String manualMode = System.getProperty(MANUAL_MODE);
if (Boolean.parseBoolean(manualMode)) {
containers.stream()
.filter(containerInfo -> !containerInfo.getQualifier().contains("balancer"))
.forEach(containerInfo -> {
log.infof("Container '%s' will be in manual mode", containerInfo.getQualifier());
containerInfo.getArquillianContainer().getContainerConfiguration().setMode("manual");
});
}
}

View file

@ -40,6 +40,8 @@ public final class SuiteContext {
private List<ContainerInfo> authServerInfo = new LinkedList<>();
private final List<List<ContainerInfo>> authServerBackendsInfo = new ArrayList<>();
private final List<ContainerInfo> cacheServersInfo = new ArrayList<>();
private ContainerInfo migratedAuthServerInfo;
private final MigrationContext migrationContext = new MigrationContext();
@ -96,6 +98,13 @@ public final class SuiteContext {
this.authServerInfo.set(dcIndex, serverInfo);
}
public void addCacheServerInfo(int dcIndex, ContainerInfo serverInfo) {
while (dcIndex >= cacheServersInfo.size()) {
cacheServersInfo.add(null);
}
this.cacheServersInfo.set(dcIndex, serverInfo);
}
public List<ContainerInfo> getAuthServerBackendsInfo() {
return getAuthServerBackendsInfo(0);
}
@ -108,6 +117,10 @@ public final class SuiteContext {
return authServerBackendsInfo;
}
public List<ContainerInfo> getCacheServersInfo() {
return cacheServersInfo;
}
public void addAuthServerBackendsInfo(int dcIndex, ContainerInfo container) {
while (dcIndex >= authServerBackendsInfo.size()) {
authServerBackendsInfo.add(new LinkedList<>());
@ -161,6 +174,10 @@ public final class SuiteContext {
int dcIndex = i;
getDcAuthServerBackendsInfo().get(i).forEach(bInfo -> sb.append("Backend (dc=").append(dcIndex).append("): ").append(bInfo).append("\n"));
}
for (int dcIndex=0 ; dcIndex<cacheServersInfo.size() ; dcIndex++) {
sb.append("CacheServer (dc=").append(dcIndex).append("): ").append(getCacheServersInfo().get(dcIndex)).append("\n");
}
} else if (isAuthServerCluster()) {
sb.append(isAuthServerCluster() ? "\nFrontend: " : "")
.append(getAuthServerInfo().getQualifier())

View file

@ -16,6 +16,7 @@
*/
package org.keycloak.testsuite.crossdc;
import org.apache.commons.io.FileUtils;
import org.keycloak.admin.client.Keycloak;
import org.keycloak.models.Constants;
import org.keycloak.testsuite.AbstractTestRealmKeycloakTest;
@ -24,6 +25,8 @@ import org.keycloak.testsuite.arquillian.LoadBalancerController;
import org.keycloak.testsuite.arquillian.annotation.LoadBalancer;
import org.keycloak.testsuite.auth.page.AuthRealm;
import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@ -321,6 +324,45 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
return dcNodes.stream().filter(c -> ! c.isManual());
}
/**
* Returns cache server corresponding to given DC
* @param dc
* @return
*/
public ContainerInfo getCacheServer(DC dc) {
int dcIndex = dc.ordinal();
return this.suiteContext.getCacheServersInfo().get(dcIndex);
}
public void stopCacheServer(ContainerInfo cacheServer) {
log.infof("Stopping %s", cacheServer.getQualifier());
containerController.stop(cacheServer.getQualifier());
// Workaround for possible arquillian bug. Needs to cleanup dir manually
String setupCleanServerBaseDir = cacheServer.getArquillianContainer().getContainerConfiguration().getContainerProperties().get("setupCleanServerBaseDir");
String cleanServerBaseDir = cacheServer.getArquillianContainer().getContainerConfiguration().getContainerProperties().get("cleanServerBaseDir");
if (Boolean.parseBoolean(setupCleanServerBaseDir)) {
log.infof("Going to clean directory: %s", cleanServerBaseDir);
File dir = new File(cleanServerBaseDir);
if (dir.exists()) {
try {
FileUtils.cleanDirectory(dir);
File deploymentsDir = new File(dir, "deployments");
deploymentsDir.mkdir();
} catch (IOException ioe) {
throw new RuntimeException("Failed to clean directory: " + cleanServerBaseDir, ioe);
}
}
}
log.infof("Stopped %s", cacheServer.getQualifier());
}
/**
* Sets time offset on all the started containers.

View file

@ -0,0 +1,187 @@
/*
* Copyright 2017 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.testsuite.crossdc.manual;
import org.junit.Test;
import org.keycloak.OAuth2Constants;
import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
import org.keycloak.testsuite.Assert;
import org.keycloak.testsuite.arquillian.AuthServerTestEnricher;
import org.keycloak.testsuite.crossdc.AbstractAdminCrossDCTest;
import org.keycloak.testsuite.crossdc.DC;
import org.keycloak.testsuite.util.OAuthClient;
/**
* Tests userSessions and offline sessions preloading at startup
*
* This test requires that lifecycle of infinispan/JDG servers is managed by testsuite, so you need to run with:
*
* -Dmanual.mode=true
*
* @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
*/
public class SessionsPreloadCrossDCTest extends AbstractAdminCrossDCTest {
private static final int SESSIONS_COUNT = 10;
@Override
public void beforeAbstractKeycloakTest() throws Exception {
// Doublecheck we are in manual mode
Assert.assertTrue("The test requires to be executed with manual.mode=true", suiteContext.getCacheServersInfo().get(0).isManual());
stopAllCacheServersAndAuthServers();
// Start DC1 only
containerController.start(getCacheServer(DC.FIRST).getQualifier());
startBackendNode(DC.FIRST, 0);
enableLoadBalancerNode(DC.FIRST, 0);
super.beforeAbstractKeycloakTest();
}
// Override as we are in manual mode
@Override
public void enableOnlyFirstNodeInFirstDc() {
}
// Override as we are in manual mode
@Override
public void terminateManuallyStartedServers() {
}
@Override
public void afterAbstractKeycloakTest() {
super.afterAbstractKeycloakTest();
// Remove realms now. In @AfterClass servers are already shutdown
AuthServerTestEnricher.removeTestRealms(testContext, adminClient);
testContext.setTestRealmReps(null);
adminClient.close();
adminClient = null;
testContext.setAdminClient(null);
stopAllCacheServersAndAuthServers();
}
private void stopAllCacheServersAndAuthServers() {
log.infof("Going to stop all auth servers");
stopBackendNode(DC.FIRST, 0);
disableLoadBalancerNode(DC.FIRST, 0);
stopBackendNode(DC.SECOND, 0);
disableLoadBalancerNode(DC.SECOND, 0);
log.infof("Auth servers stopped successfully. Going to stop all cache servers");
suiteContext.getCacheServersInfo().stream()
.filter(containerInfo -> containerInfo.isStarted())
.forEach(containerInfo -> {
stopCacheServer(containerInfo);
});
log.infof("Cache servers stopped successfully");
}
@Test
public void sessionsPreloadTest() throws Exception {
int sessionsBefore = getTestingClientForStartedNodeInDc(0).testing().cache(InfinispanConnectionProvider.SESSION_CACHE_NAME).size();
log.infof("sessionsBefore: %d", sessionsBefore);
// Create initial sessions
createInitialSessions(false);
// Start 2nd DC.
containerController.start(getCacheServer(DC.SECOND).getQualifier());
startBackendNode(DC.SECOND, 0);
enableLoadBalancerNode(DC.SECOND, 0);
// Ensure sessions are loaded in both 1st DC and 2nd DC
int sessions01 = getTestingClientForStartedNodeInDc(0).testing().cache(InfinispanConnectionProvider.SESSION_CACHE_NAME).size();
int sessions02 = getTestingClientForStartedNodeInDc(1).testing().cache(InfinispanConnectionProvider.SESSION_CACHE_NAME).size();
log.infof("sessions01: %d, sessions02: %d", sessions01, sessions02);
Assert.assertEquals(sessions01, sessionsBefore + SESSIONS_COUNT);
Assert.assertEquals(sessions02, sessionsBefore + SESSIONS_COUNT);
// On DC2 sessions were preloaded from from remoteCache
Assert.assertTrue(getTestingClientForStartedNodeInDc(1).testing().cache(InfinispanConnectionProvider.WORK_CACHE_NAME).contains("distributed::remoteCacheLoad::sessions"));
}
@Test
public void offlineSessionsPreloadTest() throws Exception {
int offlineSessionsBefore = getTestingClientForStartedNodeInDc(0).testing().cache(InfinispanConnectionProvider.OFFLINE_SESSION_CACHE_NAME).size();
log.infof("offlineSessionsBefore: %d", offlineSessionsBefore);
// Create initial sessions
createInitialSessions(true);
int offlineSessions01 = getTestingClientForStartedNodeInDc(0).testing().cache(InfinispanConnectionProvider.OFFLINE_SESSION_CACHE_NAME).size();
Assert.assertEquals(offlineSessions01, offlineSessionsBefore + SESSIONS_COUNT);
log.infof("offlineSessions01: %d", offlineSessions01);
// Stop Everything
stopAllCacheServersAndAuthServers();
// Start DC1. Sessions should be preloaded from DB
containerController.start(getCacheServer(DC.FIRST).getQualifier());
startBackendNode(DC.FIRST, 0);
enableLoadBalancerNode(DC.FIRST, 0);
// Start DC2. Sessions should be preloaded from remoteCache
containerController.start(getCacheServer(DC.SECOND).getQualifier());
startBackendNode(DC.SECOND, 0);
enableLoadBalancerNode(DC.SECOND, 0);
// Ensure sessions are loaded in both 1st DC and 2nd DC
int offlineSessions11 = getTestingClientForStartedNodeInDc(0).testing().cache(InfinispanConnectionProvider.OFFLINE_SESSION_CACHE_NAME).size();
int offlineSessions12 = getTestingClientForStartedNodeInDc(1).testing().cache(InfinispanConnectionProvider.OFFLINE_SESSION_CACHE_NAME).size();
log.infof("offlineSessions11: %d, offlineSessions12: %d", offlineSessions11, offlineSessions12);
Assert.assertEquals(offlineSessions11, offlineSessionsBefore + SESSIONS_COUNT);
Assert.assertEquals(offlineSessions12, offlineSessionsBefore + SESSIONS_COUNT);
// On DC1 sessions were preloaded from DB. On DC2 sessions were preloaded from remoteCache
Assert.assertTrue(getTestingClientForStartedNodeInDc(0).testing().cache(InfinispanConnectionProvider.WORK_CACHE_NAME).contains("distributed::offlineUserSessions"));
Assert.assertFalse(getTestingClientForStartedNodeInDc(0).testing().cache(InfinispanConnectionProvider.WORK_CACHE_NAME).contains("distributed::remoteCacheLoad::offlineSessions"));
Assert.assertFalse(getTestingClientForStartedNodeInDc(1).testing().cache(InfinispanConnectionProvider.WORK_CACHE_NAME).contains("distributed::offlineUserSessions"));
Assert.assertTrue(getTestingClientForStartedNodeInDc(1).testing().cache(InfinispanConnectionProvider.WORK_CACHE_NAME).contains("distributed::remoteCacheLoad::offlineSessions"));
}
private void createInitialSessions(boolean offline) throws Exception {
if (offline) {
oauth.scope(OAuth2Constants.OFFLINE_ACCESS);
}
for (int i=0 ; i<SESSIONS_COUNT ; i++) {
OAuthClient.AccessTokenResponse resp = oauth.doGrantAccessTokenRequest("password", "test-user@localhost", "password");
Assert.assertNull(resp.getError());
Assert.assertNotNull(resp.getAccessToken());
}
}
}