KEYCLOAK-7924 Speed-up crossdc tests

Co-Authored-By: Hynek Mlnarik <hmlnarik@redhat.com>
This commit is contained in:
vramik 2018-07-26 13:37:47 +02:00 committed by Hynek Mlnařík
parent 38017d3cec
commit ecd3fcc0af
15 changed files with 468 additions and 367 deletions

View file

@ -15,8 +15,8 @@ env:
- TESTS=server-group3
- TESTS=server-group4
- TESTS=old
- TESTS=crossdc1
- TESTS=crossdc2
- TESTS=crossdc-server
- TESTS=crossdc-adapter
jdk:
- oraclejdk8

View file

@ -249,7 +249,7 @@ public class AuthServerTestEnricher {
}
suiteContextProducer.set(suiteContext);
CacheServerTestEnricher.initializeSuiteContext(suiteContext);
CrossDCTestEnricher.initializeSuiteContext(suiteContext);
log.info("\n\n" + suiteContext);
}

View file

@ -1,90 +0,0 @@
/*
* Copyright 2018 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.testsuite.arquillian;
import java.io.File;
import java.io.IOException;
import org.apache.commons.io.FileUtils;
import org.jboss.arquillian.container.test.api.ContainerController;
import org.jboss.arquillian.core.api.Instance;
import org.jboss.arquillian.core.spi.Validate;
import org.jboss.arquillian.core.api.annotation.Inject;
import org.jboss.arquillian.core.api.annotation.Observes;
import org.jboss.arquillian.test.spi.event.suite.AfterClass;
import org.jboss.logging.Logger;
import org.keycloak.testsuite.crossdc.DC;
/**
*
* @author vramik
*/
public class CacheServerTestEnricher {
protected static final Logger log = Logger.getLogger(CacheServerTestEnricher.class);
private static SuiteContext suiteContext;
@Inject
private Instance<ContainerController> containerController;
static void initializeSuiteContext(SuiteContext suiteContext) {
Validate.notNull(suiteContext, "Suite context cannot be null.");
CacheServerTestEnricher.suiteContext = suiteContext;
}
public void afterClass(@Observes(precedence = 4) AfterClass event) {
if (!suiteContext.getCacheServersInfo().isEmpty()) {
stopCacheServer(suiteContext.getCacheServersInfo().get(DC.FIRST.ordinal()));
stopCacheServer(suiteContext.getCacheServersInfo().get(DC.SECOND.ordinal()));
}
}
private void stopCacheServer(ContainerInfo cacheServer) {
if (containerController.get().isStarted(cacheServer.getQualifier())) {
log.infof("Stopping %s", cacheServer.getQualifier());
containerController.get().stop(cacheServer.getQualifier());
// Workaround for possible arquillian bug. Needs to cleanup dir manually
String setupCleanServerBaseDir = getContainerProperty(cacheServer, "setupCleanServerBaseDir");
String cleanServerBaseDir = getContainerProperty(cacheServer, "cleanServerBaseDir");
if (Boolean.parseBoolean(setupCleanServerBaseDir)) {
log.infof("Going to clean directory: %s", cleanServerBaseDir);
File dir = new File(cleanServerBaseDir);
if (dir.exists()) {
try {
dir.renameTo(new File(dir.getParentFile(), dir.getName() + "--" + System.currentTimeMillis()));
File deploymentsDir = new File(dir, "deployments");
FileUtils.forceMkdir(deploymentsDir);
} catch (IOException ioe) {
throw new RuntimeException("Failed to clean directory: " + cleanServerBaseDir, ioe);
}
}
}
log.infof("Stopped %s", cacheServer.getQualifier());
}
}
private String getContainerProperty(ContainerInfo cacheServer, String propertyName) {
return cacheServer.getArquillianContainer().getContainerConfiguration().getContainerProperties().get(propertyName);
}
}

View file

@ -0,0 +1,329 @@
/*
* Copyright 2018 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.testsuite.arquillian;
import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.io.FileUtils;
import static org.hamcrest.Matchers.lessThan;
import org.jboss.arquillian.container.test.api.ContainerController;
import org.jboss.arquillian.core.api.Instance;
import org.jboss.arquillian.core.spi.Validate;
import org.jboss.arquillian.core.api.annotation.Inject;
import org.jboss.arquillian.core.api.annotation.Observes;
import org.jboss.arquillian.test.spi.event.suite.After;
import org.jboss.arquillian.test.spi.event.suite.AfterSuite;
import org.jboss.arquillian.test.spi.event.suite.Before;
import org.jboss.logging.Logger;
import static org.junit.Assert.assertThat;
import org.keycloak.admin.client.Keycloak;
import org.keycloak.models.Constants;
import org.keycloak.testsuite.arquillian.annotation.InitialDcState;
import org.keycloak.testsuite.auth.page.AuthRealm;
import org.keycloak.testsuite.client.KeycloakTestingClient;
import org.keycloak.testsuite.crossdc.DC;
import org.keycloak.testsuite.crossdc.ServerSetup;
import java.util.Collection;
import java.util.function.Consumer;
import org.jboss.arquillian.container.spi.event.StopSuiteContainers;
/**
*
* @author vramik
*/
public class CrossDCTestEnricher {
protected static final Logger log = Logger.getLogger(CrossDCTestEnricher.class);
private static SuiteContext suiteContext;
@Inject
private static Instance<ContainerController> containerController;
private static final Map<ContainerInfo, Keycloak> backendAdminClients = new HashMap<>();
private static final Map<ContainerInfo, KeycloakTestingClient> backendTestingClients = new HashMap<>();
static void initializeSuiteContext(SuiteContext suiteContext) {
Validate.notNull(suiteContext, "Suite context cannot be null.");
CrossDCTestEnricher.suiteContext = suiteContext;
}
public void beforeTest(@Observes(precedence = -2) Before event) {
if (!suiteContext.isAuthServerCrossDc()) return;
//if annotation is present on method
InitialDcState annotation = event.getTestMethod().getAnnotation(InitialDcState.class);
//annotation not present on method, taking it from class
if (annotation == null) {
Class<?> annotatedClass = getNearestSuperclassWithAnnotation(event.getTestClass().getJavaClass(), InitialDcState.class);
annotation = annotatedClass.getAnnotation(InitialDcState.class);
}
if (annotation == null) {
log.debug("No environment preparation requested, not changing auth/cache server run status.");
return; // Test does not specify its environment, so it's on its own
}
ServerSetup cacheServers = annotation.cacheServers();
ServerSetup authServers = annotation.authServers();
switch (cacheServers) {
case ALL_NODES_IN_EVERY_DC:
case FIRST_NODE_IN_EVERY_DC: //the same as ALL_NODES_IN_EVERY_DC as there is only one cache server per DC
case ALL_NODES_IN_FIRST_DC_FIRST_NODE_IN_SECOND_DC:
DC.validDcsStream().forEach(CrossDCTestEnricher::startCacheServer);
break;
case FIRST_NODE_IN_FIRST_DC:
case ALL_NODES_IN_FIRST_DC_NO_NODES_IN_SECOND_DC:
startCacheServer(DC.FIRST);
stopCacheServer(DC.SECOND);
break;
}
switch (authServers) {
case ALL_NODES_IN_EVERY_DC:
forAllBackendNodes(CrossDCTestEnricher::startAuthServerBackendNode);
break;
case FIRST_NODE_IN_EVERY_DC:
DC.validDcsStream().forEach((DC dc) -> startAuthServerBackendNode(dc, 0));
DC.validDcsStream().forEach((DC dc) -> stopAuthServerBackendNode(dc, 1));
break;
case FIRST_NODE_IN_FIRST_DC:
startAuthServerBackendNode(DC.FIRST, 0);
stopAuthServerBackendNode(DC.FIRST, 1);
forAllBackendNodesInDc(DC.SECOND, CrossDCTestEnricher::stopAuthServerBackendNode);
break;
case ALL_NODES_IN_FIRST_DC_FIRST_NODE_IN_SECOND_DC:
forAllBackendNodesInDc(DC.FIRST, CrossDCTestEnricher::startAuthServerBackendNode);
startAuthServerBackendNode(DC.SECOND, 0);
stopAuthServerBackendNode(DC.SECOND, 1);
break;
case ALL_NODES_IN_FIRST_DC_NO_NODES_IN_SECOND_DC:
forAllBackendNodesInDc(DC.FIRST, CrossDCTestEnricher::startAuthServerBackendNode);
forAllBackendNodesInDc(DC.SECOND, CrossDCTestEnricher::stopAuthServerBackendNode);
break;
}
suspendPeriodicTasks();
}
public void afterTest(@Observes After event) {
if (!suiteContext.isAuthServerCrossDc()) return;
restorePeriodicTasks();
}
public void stopSuiteContainers(@Observes(precedence = 4) StopSuiteContainers event) {
if (!suiteContext.isAuthServerCrossDc()) return;
DC.validDcsStream().forEach(CrossDCTestEnricher::stopCacheServer);
forAllBackendNodes(CrossDCTestEnricher::stopAuthServerBackendNode);
}
private static void createRESTClientsForNode(ContainerInfo node) {
if (!backendAdminClients.containsKey(node)) {
backendAdminClients.put(node, createAdminClientFor(node));
}
if (!backendTestingClients.containsKey(node)) {
backendTestingClients.put(node, createTestingClientFor(node));
}
}
private static void removeRESTClientsForNode(ContainerInfo node) {
if (backendAdminClients.containsKey(node)) {
backendAdminClients.get(node).close();
backendAdminClients.remove(node);
}
if (backendTestingClients.containsKey(node)) {
backendTestingClients.get(node).close();
backendTestingClients.remove(node);
}
}
public static Map<ContainerInfo, Keycloak> getBackendAdminClients() {
return Collections.unmodifiableMap(backendAdminClients);
}
public static Map<ContainerInfo, KeycloakTestingClient> getBackendTestingClients() {
return Collections.unmodifiableMap(backendTestingClients);
}
private static Keycloak createAdminClientFor(ContainerInfo node) {
log.info("--DC: Initializing admin client for " + node.getContextRoot() + "/auth");
return Keycloak.getInstance(node.getContextRoot() + "/auth", AuthRealm.MASTER, AuthRealm.ADMIN, AuthRealm.ADMIN, Constants.ADMIN_CLI_CLIENT_ID);
}
private static KeycloakTestingClient createTestingClientFor(ContainerInfo node) {
log.info("--DC: Initializing testing client for " + node.getContextRoot() + "/auth");
return KeycloakTestingClient.getInstance(node.getContextRoot() + "/auth");
}
// Disable periodic tasks in cross-dc tests. It's needed to have some scenarios more stable.
private static void suspendPeriodicTasks() {
log.debug("--DC: suspendPeriodicTasks");
backendTestingClients.values().stream().forEach((KeycloakTestingClient testingClient) -> {
testingClient.testing().suspendPeriodicTasks();
});
}
private static void restorePeriodicTasks() {
log.debug("--DC: restorePeriodicTasks");
backendTestingClients.values().stream().forEach((KeycloakTestingClient testingClient) -> {
testingClient.testing().restorePeriodicTasks();
});
}
/**
* Returns cache server corresponding to given DC
* @param dc
* @return
*/
private static ContainerInfo getCacheServer(DC dc) {
assertValidDc(dc);
int dcIndex = dc.ordinal();
return suiteContext.getCacheServersInfo().get(dcIndex);
}
private static void assertValidDc(DC dc) throws IllegalStateException {
if (dc == DC.UNDEFINED) {
throw new IllegalStateException("Invalid DC used: " + DC.UNDEFINED);
}
}
public static void startCacheServer(DC dc) {
if (!containerController.get().isStarted(getCacheServer(dc).getQualifier())) {
log.infof("--DC: Starting %s", getCacheServer(dc).getQualifier());
containerController.get().start(getCacheServer(dc).getQualifier());
log.infof("--DC: Started %s", getCacheServer(dc).getQualifier());
}
}
public static void stopCacheServer(DC dc) {
String qualifier = getCacheServer(dc).getQualifier();
if (containerController.get().isStarted(qualifier)) {
log.infof("--DC: Stopping %s", qualifier);
containerController.get().stop(qualifier);
// Workaround for possible arquillian bug. Needs to cleanup dir manually
String setupCleanServerBaseDir = getContainerProperty(getCacheServer(dc), "setupCleanServerBaseDir");
String cleanServerBaseDir = getContainerProperty(getCacheServer(dc), "cleanServerBaseDir");
if (Boolean.parseBoolean(setupCleanServerBaseDir)) {
log.debugf("Going to clean directory: %s", cleanServerBaseDir);
File dir = new File(cleanServerBaseDir);
if (dir.exists()) {
try {
dir.renameTo(new File(dir.getParentFile(), dir.getName() + "-backup-" + System.currentTimeMillis()));
File deploymentsDir = new File(dir, "deployments");
FileUtils.forceMkdir(deploymentsDir);
} catch (IOException ioe) {
throw new RuntimeException("Failed to clean directory: " + cleanServerBaseDir, ioe);
}
}
}
log.infof("--DC: Stopped %s", qualifier);
}
}
public static void forAllBackendNodes(Consumer<ContainerInfo> functionOnContainerInfo) {
suiteContext.getDcAuthServerBackendsInfo().stream()
.flatMap(Collection::stream)
.forEach(functionOnContainerInfo);
}
public static void forAllBackendNodesInDc(DC dc, Consumer<ContainerInfo> functionOnContainerInfo) {
assertValidDc(dc);
suiteContext.getDcAuthServerBackendsInfo().get(dc.ordinal()).stream()
.forEach(functionOnContainerInfo);
}
public static void stopAuthServerBackendNode(ContainerInfo containerInfo) {
if (containerInfo.isStarted()) {
log.infof("--DC: Stopping backend auth-server node: %s", containerInfo.getQualifier());
removeRESTClientsForNode(containerInfo);
containerController.get().stop(containerInfo.getQualifier());
}
}
public static void startAuthServerBackendNode(ContainerInfo containerInfo) {
if (! containerInfo.isStarted()) {
log.infof("--DC: Starting backend auth-server node: %s", containerInfo.getQualifier());
containerController.get().start(containerInfo.getQualifier());
createRESTClientsForNode(containerInfo);
}
}
public static ContainerInfo getBackendNode(DC dc, int nodeIndex) {
assertValidDc(dc);
int dcIndex = dc.ordinal();
assertThat((Integer) dcIndex, lessThan(suiteContext.getDcAuthServerBackendsInfo().size()));
final List<ContainerInfo> dcNodes = suiteContext.getDcAuthServerBackendsInfo().get(dcIndex);
assertThat((Integer) nodeIndex, lessThan(dcNodes.size()));
return dcNodes.get(nodeIndex);
}
/**
* Starts a manually-controlled backend auth-server node in cross-DC scenario.
* @param dc
* @param nodeIndex
* @return Started instance descriptor.
*/
public static ContainerInfo startAuthServerBackendNode(DC dc, int nodeIndex) {
ContainerInfo dcNode = getBackendNode(dc, nodeIndex);
startAuthServerBackendNode(dcNode);
return dcNode;
}
/**
* Stops a manually-controlled backend auth-server node in cross-DC scenario.
* @param dc
* @param nodeIndex
* @return Stopped instance descriptor.
*/
public static ContainerInfo stopAuthServerBackendNode(DC dc, int nodeIndex) {
ContainerInfo dcNode = getBackendNode(dc, nodeIndex);
stopAuthServerBackendNode(dcNode);
return dcNode;
}
private Class getNearestSuperclassWithAnnotation(Class<?> testClass, Class annotationClass) {
return (testClass.isAnnotationPresent(annotationClass)) ? testClass
: (testClass.getSuperclass().equals(Object.class) ? null // stop recursion
: getNearestSuperclassWithAnnotation(testClass.getSuperclass(), annotationClass)); // continue recursion
}
private static String getContainerProperty(ContainerInfo cacheServer, String propertyName) {
return cacheServer.getArquillianContainer().getContainerConfiguration().getContainerProperties().get(propertyName);
}
}

View file

@ -66,7 +66,7 @@ public class KeycloakArquillianExtension implements LoadableExtension {
.observer(JmxConnectorRegistryCreator.class)
.observer(AuthServerTestEnricher.class)
.observer(AppServerTestEnricher.class)
.observer(CacheServerTestEnricher.class)
.observer(CrossDCTestEnricher.class)
.observer(H2TestEnricher.class);
builder
.service(TestExecutionDecider.class, MigrationTestExecutionDecider.class)

View file

@ -0,0 +1,44 @@
/*
* Copyright 201 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.testsuite.arquillian.annotation;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.Target;
import org.keycloak.testsuite.crossdc.ServerSetup;
import static java.lang.annotation.RetentionPolicy.RUNTIME;
/**
* Specifies initial state of auth-server and cache-server nodes before start of each test
* in multinode setup like in cross-DC tests.
* When a test class is annotated, this annotation is applied to every test method in the class
* but can be overridden on method level.
*
* @author vramik
* @author hmlnarik
*/
@Documented
@Retention(RUNTIME)
@Target({ElementType.TYPE, ElementType.METHOD})
public @interface InitialDcState {
ServerSetup cacheServers() default ServerSetup.FIRST_NODE_IN_EVERY_DC;
ServerSetup authServers() default ServerSetup.FIRST_NODE_IN_EVERY_DC;
}

View file

@ -16,6 +16,9 @@
*/
package org.keycloak.testsuite.crossdc;
import java.util.Arrays;
import java.util.stream.Stream;
/**
* Identifier of datacentre in the testsuite
* @author hmlnarik
@ -28,4 +31,10 @@ public enum DC {
public int getDcIndex() {
return ordinal();
}
private static final DC[] VALID_DCS = new DC[] { FIRST, SECOND };
public static Stream<DC> validDcsStream() {
return Arrays.stream(VALID_DCS);
}
}

View file

@ -0,0 +1,28 @@
/*
* Copyright 2018 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.testsuite.crossdc;
/**
* @author vramik
*/
public enum ServerSetup {
FIRST_NODE_IN_FIRST_DC,
FIRST_NODE_IN_EVERY_DC,
ALL_NODES_IN_EVERY_DC,
ALL_NODES_IN_FIRST_DC_FIRST_NODE_IN_SECOND_DC,
ALL_NODES_IN_FIRST_DC_NO_NODES_IN_SECOND_DC;
}

View file

@ -16,38 +16,31 @@
*/
package org.keycloak.testsuite.crossdc;
import org.apache.commons.io.FileUtils;
import org.keycloak.admin.client.Keycloak;
import org.keycloak.models.Constants;
import org.keycloak.testsuite.AbstractTestRealmKeycloakTest;
import org.keycloak.testsuite.arquillian.AuthServerTestEnricher;
import org.keycloak.testsuite.arquillian.ContainerInfo;
import org.keycloak.testsuite.arquillian.LoadBalancerController;
import org.keycloak.testsuite.arquillian.annotation.LoadBalancer;
import org.keycloak.testsuite.auth.page.AuthRealm;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.jboss.arquillian.container.test.api.ContainerController;
import org.jboss.arquillian.test.api.ArquillianResource;
import org.junit.After;
import org.junit.Before;
import org.keycloak.testsuite.client.KeycloakTestingClient;
import static org.hamcrest.Matchers.lessThan;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import org.keycloak.representations.idm.RealmRepresentation;
import org.keycloak.testsuite.arquillian.CrossDCTestEnricher;
import org.keycloak.testsuite.arquillian.annotation.InitialDcState;
/**
* Abstract cross-data-centre test that defines primitives for handling cross-DC setup.
* @author hmlnarik
*/
@InitialDcState
public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest {
// Keep the following constants in sync with arquillian
@ -59,59 +52,34 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
@LoadBalancer(value = QUALIFIER_NODE_BALANCER)
protected LoadBalancerController loadBalancerCtrl;
@ArquillianResource
protected ContainerController containerController;
protected Map<ContainerInfo, Keycloak> backendAdminClients = new HashMap<>();
protected Map<ContainerInfo, KeycloakTestingClient> backendTestingClients = new HashMap<>();
@Before
@Override
public void beforeAbstractKeycloakTest() throws Exception {
log.debug("--DC: Starting both cache servers and first node from each DC");
startCacheServer(DC.FIRST);
startCacheServer(DC.SECOND);
startBackendNode(DC.FIRST, 0);
startBackendNode(DC.SECOND, 0);
initRESTClientsForStartedNodes();
suspendPeriodicTasks();
enableOnlyFirstNodeInFirstDc();
super.beforeAbstractKeycloakTest();
}
@Override
public void deleteCookies() {
//Overrides AbstractTestRealmKeycloakTest.deleteCookies
//as it tries to delete cookies in 'test' realm @After test
//when backend containers are stopped already.
}
@After
@Override
public void afterAbstractKeycloakTest() {
log.debug("--DC: after AbstractCrossDCTest");
CrossDCTestEnricher.startAuthServerBackendNode(DC.FIRST, 0); // make sure first node is started
enableOnlyFirstNodeInFirstDc();
super.afterAbstractKeycloakTest();
restorePeriodicTasks();
removeTestRealms();
terminateStartedServers();
loadBalancerCtrl.disableAllBackendNodes();
}
private void enableOnlyFirstNodeInFirstDc() {
log.debug("--DC: Enable only first node in first datacenter");
this.loadBalancerCtrl.disableAllBackendNodes();
if (!getBackendNode(DC.FIRST, 0).isStarted()) {
if (!CrossDCTestEnricher.getBackendNode(DC.FIRST, 0).isStarted()) {
throw new IllegalStateException("--DC: Trying to enable not started node on load-balancer");
}
loadBalancerCtrl.enableBackendNodeByName(getBackendNode(DC.FIRST, 0).getQualifier());
loadBalancerCtrl.enableBackendNodeByName(CrossDCTestEnricher.getBackendNode(DC.FIRST, 0).getQualifier());
}
private void removeTestRealms() {
@ -122,52 +90,6 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
testContext.setTestRealmReps(new ArrayList<>());
}
protected void terminateStartedServers() {
log.debug("--DC: Halting all nodes that are started");
this.suiteContext.getDcAuthServerBackendsInfo().stream()
.flatMap(List::stream)
.filter(ContainerInfo::isStarted)
.forEach((ContainerInfo containerInfo) -> {
containerController.stop(containerInfo.getQualifier());
removeRESTClientsForNode(containerInfo);
});
}
private void initRESTClientsForStartedNodes() {
log.debug("--DC: Init REST clients for started nodes");
this.suiteContext.getDcAuthServerBackendsInfo().stream()
.flatMap(List::stream)
.filter(ContainerInfo::isStarted)
.forEach(containerInfo -> {
createRESTClientsForNode(containerInfo);
});
}
// Disable periodic tasks in cross-dc tests. It's needed to have some scenarios more stable.
private void suspendPeriodicTasks() {
log.debug("--DC: suspendPeriodicTasks");
backendTestingClients.values().stream().forEach((KeycloakTestingClient testingClient) -> {
testingClient.testing().suspendPeriodicTasks();
});
}
private void restorePeriodicTasks() {
log.debug("--DC: restorePeriodicTasks");
backendTestingClients.values().stream().forEach((KeycloakTestingClient testingClient) -> {
testingClient.testing().restorePeriodicTasks();
});
}
protected Keycloak createAdminClientFor(ContainerInfo node) {
log.info("--DC: Initializing admin client for " + node.getContextRoot() + "/auth");
return Keycloak.getInstance(node.getContextRoot() + "/auth", AuthRealm.MASTER, AuthRealm.ADMIN, AuthRealm.ADMIN, Constants.ADMIN_CLI_CLIENT_ID);
}
protected KeycloakTestingClient createTestingClientFor(ContainerInfo node) {
log.info("--DC: Initializing testing client for " + node.getContextRoot() + "/auth");
return KeycloakTestingClient.getInstance(node.getContextRoot() + "/auth");
}
protected Keycloak getAdminClientForStartedNodeInDc(int dcIndex) {
ContainerInfo firstStartedNode = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex).stream()
.filter(ContainerInfo::isStarted)
@ -182,14 +104,13 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
* @return
*/
protected Keycloak getAdminClientFor(ContainerInfo node) {
Keycloak client = backendAdminClients.get(node);
Keycloak client = CrossDCTestEnricher.getBackendAdminClients().get(node);
if (client == null && node.equals(suiteContext.getAuthServerInfo())) {
client = this.adminClient;
}
return client;
}
protected KeycloakTestingClient getTestingClientForStartedNodeInDc(int dcIndex) {
ContainerInfo firstStartedNode = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex).stream()
.filter(ContainerInfo::isStarted)
@ -204,35 +125,13 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
* @return
*/
protected KeycloakTestingClient getTestingClientFor(ContainerInfo node) {
KeycloakTestingClient client = backendTestingClients.get(node);
KeycloakTestingClient client = CrossDCTestEnricher.getBackendTestingClients().get(node);
if (client == null && node.equals(suiteContext.getAuthServerInfo())) {
client = this.testingClient;
}
return client;
}
protected void createRESTClientsForNode(ContainerInfo node) {
if (!backendAdminClients.containsKey(node)) {
backendAdminClients.put(node, createAdminClientFor(node));
}
if (!backendTestingClients.containsKey(node)) {
backendTestingClients.put(node, createTestingClientFor(node));
}
}
protected void removeRESTClientsForNode(ContainerInfo node) {
if (backendAdminClients.containsKey(node)) {
backendAdminClients.get(node).close();
backendAdminClients.remove(node);
}
if (backendTestingClients.containsKey(node)) {
backendTestingClients.get(node).close();
backendTestingClients.remove(node);
}
}
/**
* Disables routing requests to the given data center in the load balancer.
* @param dc
@ -293,97 +192,6 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
loadBalancerCtrl.enableBackendNodeByName(backendNode.getQualifier());
}
/**
* Starts a manually-controlled backend auth-server node in cross-DC scenario.
* @param dc
* @param nodeIndex
* @return Started instance descriptor.
*/
protected ContainerInfo startBackendNode(DC dc, int nodeIndex) {
ContainerInfo dcNode = getBackendNode(dc, nodeIndex);
assertTrue("Node " + dcNode.getQualifier() + " has to be controlled manually", dcNode.isManual());
if (!containerController.isStarted(dcNode.getQualifier())) {
log.infof("--DC: Starting backend node: %s (dcIndex: %d, nodeIndex: %d)", dcNode.getQualifier(), dc.ordinal(), nodeIndex);
containerController.start(dcNode.getQualifier());
createRESTClientsForNode(dcNode);
}
return dcNode;
}
/**
* Stops a manually-controlled backend auth-server node in cross-DC scenario.
* @param dc
* @param nodeIndex
* @return Stopped instance descriptor.
*/
protected ContainerInfo stopBackendNode(DC dc, int nodeIndex) {
ContainerInfo dcNode = getBackendNode(dc, nodeIndex);
removeRESTClientsForNode(dcNode);
assertTrue("Node " + dcNode.getQualifier() + " has to be controlled manually", dcNode.isManual());
log.infof("--DC: Stopping backend node: %s (dcIndex: %d, nodeIndex: %d)", dcNode.getQualifier(), dc.ordinal(), nodeIndex);
containerController.stop(dcNode.getQualifier());
return dcNode;
}
private ContainerInfo getBackendNode(DC dc, int nodeIndex) {
int dcIndex = dc.ordinal();
assertThat((Integer) dcIndex, lessThan(this.suiteContext.getDcAuthServerBackendsInfo().size()));
final List<ContainerInfo> dcNodes = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex);
assertThat((Integer) nodeIndex, lessThan(dcNodes.size()));
return dcNodes.get(nodeIndex);
}
/**
* Returns cache server corresponding to given DC
* @param dc
* @return
*/
protected ContainerInfo getCacheServer(DC dc) {
int dcIndex = dc.ordinal();
return this.suiteContext.getCacheServersInfo().get(dcIndex);
}
protected void startCacheServer(DC dc) {
if (!containerController.isStarted(getCacheServer(dc).getQualifier())) {
log.infof("--DC: Starting %s", getCacheServer(dc).getQualifier());
containerController.start(getCacheServer(dc).getQualifier());
}
}
protected void stopCacheServer(ContainerInfo cacheServer) {
log.infof("--DC: Stopping %s", cacheServer.getQualifier());
containerController.stop(cacheServer.getQualifier());
// Workaround for possible arquillian bug. Needs to cleanup dir manually
String setupCleanServerBaseDir = cacheServer.getArquillianContainer().getContainerConfiguration().getContainerProperties().get("setupCleanServerBaseDir");
String cleanServerBaseDir = cacheServer.getArquillianContainer().getContainerConfiguration().getContainerProperties().get("cleanServerBaseDir");
if (Boolean.parseBoolean(setupCleanServerBaseDir)) {
log.infof("--DC: Going to clean directory: %s", cleanServerBaseDir);
File dir = new File(cleanServerBaseDir);
if (dir.exists()) {
try {
dir.renameTo(new File(dir.getParentFile(), dir.getName() + "--" + System.currentTimeMillis()));
File deploymentsDir = new File(dir, "deployments");
FileUtils.forceMkdir(deploymentsDir);
} catch (IOException ioe) {
throw new RuntimeException("Failed to clean directory: " + cleanServerBaseDir, ioe);
}
}
}
log.infof("--DC: Stopped %s", cacheServer.getQualifier());
}
/**
* Sets time offset on all the started containers.
*
@ -396,7 +204,7 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
}
private void setTimeOffsetOnAllStartedContainers(int offset) {
backendTestingClients.entrySet().stream()
CrossDCTestEnricher.getBackendTestingClients().entrySet().stream()
.filter(testingClientEntry -> testingClientEntry.getKey().isStarted())
.map(testingClientEntry -> testingClientEntry.getValue())
.forEach(testingClient -> testingClient.testing().setTimeOffset(Collections.singletonMap("offset", String.valueOf(offset))));

View file

@ -50,6 +50,8 @@ import org.hamcrest.Matchers;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.greaterThan;
import static org.junit.Assert.assertThat;
import org.keycloak.testsuite.arquillian.CrossDCTestEnricher;
import org.keycloak.testsuite.arquillian.annotation.InitialDcState;
/**
*
@ -80,6 +82,7 @@ public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest {
}
@Test
@InitialDcState(authServers = ServerSetup.ALL_NODES_IN_FIRST_DC_FIRST_NODE_IN_SECOND_DC)
public void sendResetPasswordEmailSuccessWorksInCrossDc(
@JmxInfinispanCacheStatistics(dc=DC.FIRST, dcNodeIndex=0, cacheName=InfinispanConnectionProvider.ACTION_TOKEN_CACHE) InfinispanStatistics cacheDc0Node0Statistics,
@JmxInfinispanCacheStatistics(dc=DC.FIRST, dcNodeIndex=1, cacheName=InfinispanConnectionProvider.ACTION_TOKEN_CACHE) InfinispanStatistics cacheDc0Node1Statistics,
@ -87,7 +90,6 @@ public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest {
@JmxInfinispanChannelStatistics() InfinispanStatistics channelStatisticsCrossDc) throws Exception {
log.debug("--DC: START sendResetPasswordEmailSuccessWorksInCrossDc");
startBackendNode(DC.FIRST, 1);
cacheDc0Node1Statistics.waitToBecomeAvailable(10, TimeUnit.SECONDS);
Comparable originalNumberOfEntries = cacheDc0Node0Statistics.getSingleStatistics(Constants.STAT_CACHE_NUMBER_OF_ENTRIES);
@ -155,6 +157,7 @@ public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest {
}
@Test
@InitialDcState(authServers = ServerSetup.FIRST_NODE_IN_FIRST_DC)
public void sendResetPasswordEmailAfterNewNodeAdded() throws IOException, MessagingException {
log.debug("--DC: START sendResetPasswordEmailAfterNewNodeAdded");
disableDcOnLoadBalancer(DC.SECOND);
@ -188,7 +191,8 @@ public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest {
assertEquals("Your account has been updated.", PageUtils.getPageTitle(driver));
disableDcOnLoadBalancer(DC.FIRST);
startBackendNode(DC.SECOND, 1);
CrossDCTestEnricher.startAuthServerBackendNode(DC.SECOND, 1);
CrossDCTestEnricher.stopAuthServerBackendNode(DC.FIRST, 0);
enableLoadBalancerNode(DC.SECOND, 1);
Retry.execute(() -> {

View file

@ -37,10 +37,12 @@ import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.LaxRedirectStrategy;
import org.junit.Test;
import org.keycloak.testsuite.arquillian.annotation.InitialDcState;
/**
* @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
*/
@InitialDcState(authServers = ServerSetup.ALL_NODES_IN_EVERY_DC)
public class ConcurrentLoginCrossDCTest extends ConcurrentLoginTest {
@ArquillianResource
@ -56,35 +58,11 @@ public class ConcurrentLoginCrossDCTest extends ConcurrentLoginTest {
@Override
public void beforeAbstractKeycloakTestRealmImport() {
log.debug("--DC: Starting cacheServers if not started already");
suiteContext.getCacheServersInfo().stream()
.filter((containerInfo) -> !containerInfo.isStarted())
.map(ContainerInfo::getQualifier)
.forEach(containerController::start);
log.debug("--DC: Initializing load balancer - enabling all started nodes across DCs");
this.loadBalancerCtrl.disableAllBackendNodes();
this.suiteContext.getDcAuthServerBackendsInfo().stream()
.flatMap(List::stream)
.filter((containerInfo) -> !containerInfo.getQualifier().contains("manual"))
.filter((containerInfo) -> !containerInfo.isStarted())
.map(ContainerInfo::getQualifier)
.forEach((nodeName) -> {
containerController.start(nodeName);
loadBalancerCtrl.enableBackendNodeByName(nodeName);
});
loadBalancerCtrl.enableAllBackendNodes();
}
@Override
public void postAfterAbstractKeycloak() {
log.debug("--DC: postAfterAbstractKeycloak");
suiteContext.getDcAuthServerBackendsInfo().stream()
.flatMap(List::stream)
.filter(ContainerInfo::isStarted)
.map(ContainerInfo::getQualifier)
.forEach(containerController::stop);
loadBalancerCtrl.disableAllBackendNodes();
//realms is already removed and this prevents another removal in AuthServerTestEnricher.afterClass

View file

@ -40,7 +40,9 @@ import org.keycloak.representations.idm.UserSessionRepresentation;
import org.keycloak.testsuite.Assert;
import org.keycloak.common.util.Retry;
import org.keycloak.testsuite.admin.ApiUtil;
import org.keycloak.testsuite.arquillian.CrossDCTestEnricher;
import org.keycloak.testsuite.arquillian.InfinispanStatistics;
import org.keycloak.testsuite.arquillian.annotation.InitialDcState;
import org.keycloak.testsuite.arquillian.annotation.JmxInfinispanCacheStatistics;
import org.keycloak.testsuite.arquillian.annotation.JmxInfinispanChannelStatistics;
import org.keycloak.testsuite.util.ClientBuilder;
@ -417,20 +419,18 @@ public class SessionExpirationCrossDCTest extends AbstractAdminCrossDCTest {
@Test
@InitialDcState(authServers = ServerSetup.ALL_NODES_IN_FIRST_DC_FIRST_NODE_IN_SECOND_DC)
public void testLogoutUserWithFailover(
@JmxInfinispanCacheStatistics(dc=DC.FIRST, managementPortProperty = "cache.server.management.port", cacheName=InfinispanConnectionProvider.USER_SESSION_CACHE_NAME) InfinispanStatistics cacheDc1Statistics,
@JmxInfinispanCacheStatistics(dc=DC.SECOND, managementPortProperty = "cache.server.2.management.port", cacheName=InfinispanConnectionProvider.USER_SESSION_CACHE_NAME) InfinispanStatistics cacheDc2Statistics,
@JmxInfinispanChannelStatistics() InfinispanStatistics channelStatisticsCrossDc) throws Exception {
// Start node2 on first DC
startBackendNode(DC.FIRST, 1);
// Don't include remote stats. Size is smaller because of distributed cache
List<OAuthClient.AccessTokenResponse> responses = createInitialSessions(InfinispanConnectionProvider.USER_SESSION_CACHE_NAME, InfinispanConnectionProvider.CLIENT_SESSION_CACHE_NAME,
false, cacheDc1Statistics, cacheDc2Statistics, false);
// Kill node2 now. Around 10 sessions (half of SESSIONS_COUNT) will be lost on Keycloak side. But not on infinispan side
stopBackendNode(DC.FIRST, 1);
CrossDCTestEnricher.stopAuthServerBackendNode(DC.FIRST, 1);
channelStatisticsCrossDc.reset();
@ -461,15 +461,12 @@ public class SessionExpirationCrossDCTest extends AbstractAdminCrossDCTest {
@Test
@InitialDcState(authServers = ServerSetup.ALL_NODES_IN_EVERY_DC)
public void testLogoutWithAllStartedNodes(
@JmxInfinispanCacheStatistics(dc=DC.FIRST, managementPortProperty = "cache.server.management.port", cacheName=InfinispanConnectionProvider.USER_SESSION_CACHE_NAME) InfinispanStatistics cacheDc1Statistics,
@JmxInfinispanCacheStatistics(dc=DC.SECOND, managementPortProperty = "cache.server.2.management.port", cacheName=InfinispanConnectionProvider.USER_SESSION_CACHE_NAME) InfinispanStatistics cacheDc2Statistics,
@JmxInfinispanChannelStatistics() InfinispanStatistics channelStatisticsCrossDc) throws Exception {
// Start node2 on every DC
startBackendNode(DC.FIRST, 1);
startBackendNode(DC.SECOND, 1);
// Create sessions. Don't include remote stats. Size is smaller because of distributed cache
List<OAuthClient.AccessTokenResponse> responses = createInitialSessions(InfinispanConnectionProvider.USER_SESSION_CACHE_NAME, InfinispanConnectionProvider.CLIENT_SESSION_CACHE_NAME,
false, cacheDc1Statistics, cacheDc2Statistics, false);
@ -505,8 +502,8 @@ public class SessionExpirationCrossDCTest extends AbstractAdminCrossDCTest {
}, 50, 50);
// Stop both nodes
stopBackendNode(DC.FIRST, 1);
stopBackendNode(DC.SECOND, 1);
CrossDCTestEnricher.stopAuthServerBackendNode(DC.FIRST, 1);
CrossDCTestEnricher.stopAuthServerBackendNode(DC.SECOND, 1);
}
private void assertTestAppActiveSessionsCount(int expectedSessionsCount) {

View file

@ -23,10 +23,13 @@ import java.util.List;
import org.junit.Before;
import org.junit.Test;
import org.keycloak.OAuth2Constants;
import org.keycloak.common.util.Retry;
import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
import org.keycloak.representations.idm.RealmRepresentation;
import org.keycloak.testsuite.Assert;
import org.keycloak.testsuite.admin.ApiUtil;
import org.keycloak.testsuite.arquillian.CrossDCTestEnricher;
import org.keycloak.testsuite.arquillian.annotation.InitialDcState;
import org.keycloak.testsuite.util.OAuthClient;
/**
@ -34,38 +37,28 @@ import org.keycloak.testsuite.util.OAuthClient;
*
* @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
*/
@InitialDcState(authServers = ServerSetup.ALL_NODES_IN_FIRST_DC_NO_NODES_IN_SECOND_DC)
public class SessionsPreloadCrossDCTest extends AbstractAdminCrossDCTest {
private static final int SESSIONS_COUNT = 10;
@Before
public void beforeSessionsPreloadCrossDCTest() throws Exception {
// Start DC1 and only All Keycloak nodes on DC2 are stopped
stopBackendNode(DC.SECOND, 0);
disableDcOnLoadBalancer(DC.SECOND);
}
private void stopAllCacheServersAndAuthServers() {
log.infof("Going to stop all auth servers");
stopBackendNode(DC.FIRST, 0);
disableLoadBalancerNode(DC.FIRST, 0);
stopBackendNode(DC.SECOND, 0);
disableLoadBalancerNode(DC.SECOND, 0);
CrossDCTestEnricher.forAllBackendNodes(CrossDCTestEnricher::stopAuthServerBackendNode);
loadBalancerCtrl.disableAllBackendNodes();
log.infof("Auth servers stopped successfully. Going to stop all cache servers");
suiteContext.getCacheServersInfo().stream()
.filter(containerInfo -> containerInfo.isStarted())
.forEach(containerInfo -> {
stopCacheServer(containerInfo);
});
DC.validDcsStream().forEach(CrossDCTestEnricher::stopCacheServer);
log.infof("Cache servers stopped successfully");
}
@Test
public void sessionsPreloadTest() throws Exception {
int sessionsBefore = getTestingClientForStartedNodeInDc(0).testing().cache(InfinispanConnectionProvider.USER_SESSION_CACHE_NAME).size();
@ -75,7 +68,7 @@ public class SessionsPreloadCrossDCTest extends AbstractAdminCrossDCTest {
List<OAuthClient.AccessTokenResponse> tokenResponses = createInitialSessions(false);
// Start 2nd DC.
startBackendNode(DC.SECOND, 0);
CrossDCTestEnricher.startAuthServerBackendNode(DC.SECOND, 0);
enableLoadBalancerNode(DC.SECOND, 0);
// Ensure sessions are loaded in both 1st DC and 2nd DC
@ -113,15 +106,14 @@ public class SessionsPreloadCrossDCTest extends AbstractAdminCrossDCTest {
stopAllCacheServersAndAuthServers();
// Start cache containers on both DC1 and DC2
startCacheServer(DC.FIRST);
startCacheServer(DC.SECOND);
DC.validDcsStream().forEach(CrossDCTestEnricher::startCacheServer);
// Start Keycloak on DC1. Sessions should be preloaded from DB
startBackendNode(DC.FIRST, 0);
CrossDCTestEnricher.startAuthServerBackendNode(DC.FIRST, 0);
enableLoadBalancerNode(DC.FIRST, 0);
// Start Keycloak on DC2. Sessions should be preloaded from remoteCache
startBackendNode(DC.SECOND, 0);
CrossDCTestEnricher.startAuthServerBackendNode(DC.SECOND, 0);
enableLoadBalancerNode(DC.SECOND, 0);
// Ensure sessions are loaded in both 1st DC and 2nd DC
@ -167,9 +159,10 @@ public class SessionsPreloadCrossDCTest extends AbstractAdminCrossDCTest {
}
// Start 2nd DC.
startBackendNode(DC.SECOND, 0);
CrossDCTestEnricher.startAuthServerBackendNode(DC.SECOND, 0);
enableLoadBalancerNode(DC.SECOND, 0);
Retry.execute(() -> {
// Ensure loginFailures are loaded in both 1st DC and 2nd DC
int size1 = getTestingClientForStartedNodeInDc(0).testing().cache(InfinispanConnectionProvider.LOGIN_FAILURE_CACHE_NAME).size();
int size2 = getTestingClientForStartedNodeInDc(1).testing().cache(InfinispanConnectionProvider.LOGIN_FAILURE_CACHE_NAME).size();
@ -180,6 +173,7 @@ public class SessionsPreloadCrossDCTest extends AbstractAdminCrossDCTest {
Assert.assertEquals(size2, 1);
Assert.assertEquals(loginFailures1, loginFailuresBefore + SESSIONS_COUNT);
Assert.assertEquals(loginFailures2, loginFailuresBefore + SESSIONS_COUNT);
}, 3, 400);
// On DC2 sessions were preloaded from from remoteCache
Assert.assertTrue(getTestingClientForStartedNodeInDc(1).testing().cache(InfinispanConnectionProvider.WORK_CACHE_NAME).contains("distributed::remoteCacheLoad::loginFailures"));

View file

@ -371,7 +371,7 @@
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<id>clean-second-cache-server-arquillian-bug-workaround</id>
<id>clean-second-cache-server-arquillian-bug-workaround</id><!--https://issues.jboss.org/browse/WFARQ-44-->
<phase>process-test-resources</phase>
<goals><goal>run</goal></goals>
<configuration>

View file

@ -74,7 +74,7 @@ if [ $1 == "server-group4" ]; then
run-server-tests org.keycloak.testsuite.k*.**.*Test,org.keycloak.testsuite.m*.**.*Test,org.keycloak.testsuite.o*.**.*Test,org.keycloak.testsuite.s*.**.*Test
fi
if [ $1 == "crossdc1" ]; then
if [ $1 == "crossdc-server" ]; then
cd testsuite/integration-arquillian
mvn install -B -nsu -Pauth-servers-crossdc-jboss,auth-server-wildfly,cache-server-infinispan -DskipTests
@ -84,7 +84,7 @@ if [ $1 == "crossdc1" ]; then
exit ${PIPESTATUS[0]}
fi
if [ $1 == "crossdc2" ]; then
if [ $1 == "crossdc-adapter" ]; then
cd testsuite/integration-arquillian
mvn install -B -nsu -Pauth-servers-crossdc-jboss,auth-server-wildfly,cache-server-infinispan,app-server-wildfly -DskipTests