From ecd3fcc0af27072a0c73a8e1b027d4333bee79df Mon Sep 17 00:00:00 2001 From: vramik Date: Thu, 26 Jul 2018 13:37:47 +0200 Subject: [PATCH] KEYCLOAK-7924 Speed-up crossdc tests Co-Authored-By: Hynek Mlnarik --- .travis.yml | 4 +- .../arquillian/AuthServerTestEnricher.java | 2 +- .../arquillian/CacheServerTestEnricher.java | 90 ----- .../arquillian/CrossDCTestEnricher.java | 329 ++++++++++++++++++ .../KeycloakArquillianExtension.java | 2 +- .../arquillian/annotation/InitialDcState.java | 44 +++ .../org/keycloak/testsuite/crossdc/DC.java | 9 + .../testsuite/crossdc/ServerSetup.java | 28 ++ .../crossdc/AbstractCrossDCTest.java | 214 +----------- .../crossdc/ActionTokenCrossDCTest.java | 8 +- .../crossdc/ConcurrentLoginCrossDCTest.java | 28 +- .../crossdc/SessionExpirationCrossDCTest.java | 17 +- .../crossdc/SessionsPreloadCrossDCTest.java | 54 ++- .../integration-arquillian/tests/pom.xml | 2 +- travis-run-tests.sh | 4 +- 15 files changed, 468 insertions(+), 367 deletions(-) delete mode 100644 testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/CacheServerTestEnricher.java create mode 100644 testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/CrossDCTestEnricher.java create mode 100644 testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/annotation/InitialDcState.java create mode 100644 testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/crossdc/ServerSetup.java diff --git a/.travis.yml b/.travis.yml index 8cce1f34af..19eccf2f2e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,8 +15,8 @@ env: - TESTS=server-group3 - TESTS=server-group4 - TESTS=old - - TESTS=crossdc1 - - TESTS=crossdc2 + - TESTS=crossdc-server + - TESTS=crossdc-adapter jdk: - oraclejdk8 diff --git a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/AuthServerTestEnricher.java b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/AuthServerTestEnricher.java index b619bf8949..fb9b7e4a9c 100644 --- a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/AuthServerTestEnricher.java +++ b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/AuthServerTestEnricher.java @@ -249,7 +249,7 @@ public class AuthServerTestEnricher { } suiteContextProducer.set(suiteContext); - CacheServerTestEnricher.initializeSuiteContext(suiteContext); + CrossDCTestEnricher.initializeSuiteContext(suiteContext); log.info("\n\n" + suiteContext); } diff --git a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/CacheServerTestEnricher.java b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/CacheServerTestEnricher.java deleted file mode 100644 index cb4f49fa83..0000000000 --- a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/CacheServerTestEnricher.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright 2018 Red Hat, Inc. and/or its affiliates - * and other contributors as indicated by the @author tags. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.keycloak.testsuite.arquillian; - -import java.io.File; -import java.io.IOException; - -import org.apache.commons.io.FileUtils; -import org.jboss.arquillian.container.test.api.ContainerController; -import org.jboss.arquillian.core.api.Instance; -import org.jboss.arquillian.core.spi.Validate; -import org.jboss.arquillian.core.api.annotation.Inject; -import org.jboss.arquillian.core.api.annotation.Observes; -import org.jboss.arquillian.test.spi.event.suite.AfterClass; -import org.jboss.logging.Logger; - -import org.keycloak.testsuite.crossdc.DC; - -/** - * - * @author vramik - */ -public class CacheServerTestEnricher { - - protected static final Logger log = Logger.getLogger(CacheServerTestEnricher.class); - private static SuiteContext suiteContext; - - @Inject - private Instance containerController; - - static void initializeSuiteContext(SuiteContext suiteContext) { - Validate.notNull(suiteContext, "Suite context cannot be null."); - CacheServerTestEnricher.suiteContext = suiteContext; - } - - public void afterClass(@Observes(precedence = 4) AfterClass event) { - if (!suiteContext.getCacheServersInfo().isEmpty()) { - stopCacheServer(suiteContext.getCacheServersInfo().get(DC.FIRST.ordinal())); - stopCacheServer(suiteContext.getCacheServersInfo().get(DC.SECOND.ordinal())); - } - } - - private void stopCacheServer(ContainerInfo cacheServer) { - if (containerController.get().isStarted(cacheServer.getQualifier())) { - log.infof("Stopping %s", cacheServer.getQualifier()); - - containerController.get().stop(cacheServer.getQualifier()); - - // Workaround for possible arquillian bug. Needs to cleanup dir manually - String setupCleanServerBaseDir = getContainerProperty(cacheServer, "setupCleanServerBaseDir"); - String cleanServerBaseDir = getContainerProperty(cacheServer, "cleanServerBaseDir"); - - if (Boolean.parseBoolean(setupCleanServerBaseDir)) { - log.infof("Going to clean directory: %s", cleanServerBaseDir); - - File dir = new File(cleanServerBaseDir); - if (dir.exists()) { - try { - dir.renameTo(new File(dir.getParentFile(), dir.getName() + "--" + System.currentTimeMillis())); - - File deploymentsDir = new File(dir, "deployments"); - FileUtils.forceMkdir(deploymentsDir); - } catch (IOException ioe) { - throw new RuntimeException("Failed to clean directory: " + cleanServerBaseDir, ioe); - } - } - } - - log.infof("Stopped %s", cacheServer.getQualifier()); - } - } - - private String getContainerProperty(ContainerInfo cacheServer, String propertyName) { - return cacheServer.getArquillianContainer().getContainerConfiguration().getContainerProperties().get(propertyName); - } -} diff --git a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/CrossDCTestEnricher.java b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/CrossDCTestEnricher.java new file mode 100644 index 0000000000..a098af1fe4 --- /dev/null +++ b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/CrossDCTestEnricher.java @@ -0,0 +1,329 @@ +/* + * Copyright 2018 Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.keycloak.testsuite.arquillian; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.io.FileUtils; +import static org.hamcrest.Matchers.lessThan; +import org.jboss.arquillian.container.test.api.ContainerController; +import org.jboss.arquillian.core.api.Instance; +import org.jboss.arquillian.core.spi.Validate; +import org.jboss.arquillian.core.api.annotation.Inject; +import org.jboss.arquillian.core.api.annotation.Observes; +import org.jboss.arquillian.test.spi.event.suite.After; +import org.jboss.arquillian.test.spi.event.suite.AfterSuite; +import org.jboss.arquillian.test.spi.event.suite.Before; +import org.jboss.logging.Logger; +import static org.junit.Assert.assertThat; +import org.keycloak.admin.client.Keycloak; +import org.keycloak.models.Constants; +import org.keycloak.testsuite.arquillian.annotation.InitialDcState; +import org.keycloak.testsuite.auth.page.AuthRealm; +import org.keycloak.testsuite.client.KeycloakTestingClient; +import org.keycloak.testsuite.crossdc.DC; +import org.keycloak.testsuite.crossdc.ServerSetup; +import java.util.Collection; +import java.util.function.Consumer; +import org.jboss.arquillian.container.spi.event.StopSuiteContainers; + +/** + * + * @author vramik + */ +public class CrossDCTestEnricher { + + protected static final Logger log = Logger.getLogger(CrossDCTestEnricher.class); + private static SuiteContext suiteContext; + + @Inject + private static Instance containerController; + + private static final Map backendAdminClients = new HashMap<>(); + private static final Map backendTestingClients = new HashMap<>(); + + static void initializeSuiteContext(SuiteContext suiteContext) { + Validate.notNull(suiteContext, "Suite context cannot be null."); + CrossDCTestEnricher.suiteContext = suiteContext; + } + + public void beforeTest(@Observes(precedence = -2) Before event) { + if (!suiteContext.isAuthServerCrossDc()) return; + + //if annotation is present on method + InitialDcState annotation = event.getTestMethod().getAnnotation(InitialDcState.class); + + //annotation not present on method, taking it from class + if (annotation == null) { + Class annotatedClass = getNearestSuperclassWithAnnotation(event.getTestClass().getJavaClass(), InitialDcState.class); + + annotation = annotatedClass.getAnnotation(InitialDcState.class); + } + + if (annotation == null) { + log.debug("No environment preparation requested, not changing auth/cache server run status."); + return; // Test does not specify its environment, so it's on its own + } + + ServerSetup cacheServers = annotation.cacheServers(); + ServerSetup authServers = annotation.authServers(); + + switch (cacheServers) { + case ALL_NODES_IN_EVERY_DC: + case FIRST_NODE_IN_EVERY_DC: //the same as ALL_NODES_IN_EVERY_DC as there is only one cache server per DC + case ALL_NODES_IN_FIRST_DC_FIRST_NODE_IN_SECOND_DC: + DC.validDcsStream().forEach(CrossDCTestEnricher::startCacheServer); + break; + + case FIRST_NODE_IN_FIRST_DC: + case ALL_NODES_IN_FIRST_DC_NO_NODES_IN_SECOND_DC: + startCacheServer(DC.FIRST); + stopCacheServer(DC.SECOND); + break; + } + + switch (authServers) { + case ALL_NODES_IN_EVERY_DC: + forAllBackendNodes(CrossDCTestEnricher::startAuthServerBackendNode); + break; + case FIRST_NODE_IN_EVERY_DC: + DC.validDcsStream().forEach((DC dc) -> startAuthServerBackendNode(dc, 0)); + DC.validDcsStream().forEach((DC dc) -> stopAuthServerBackendNode(dc, 1)); + break; + + case FIRST_NODE_IN_FIRST_DC: + startAuthServerBackendNode(DC.FIRST, 0); + stopAuthServerBackendNode(DC.FIRST, 1); + forAllBackendNodesInDc(DC.SECOND, CrossDCTestEnricher::stopAuthServerBackendNode); + break; + + case ALL_NODES_IN_FIRST_DC_FIRST_NODE_IN_SECOND_DC: + forAllBackendNodesInDc(DC.FIRST, CrossDCTestEnricher::startAuthServerBackendNode); + startAuthServerBackendNode(DC.SECOND, 0); + stopAuthServerBackendNode(DC.SECOND, 1); + break; + + case ALL_NODES_IN_FIRST_DC_NO_NODES_IN_SECOND_DC: + forAllBackendNodesInDc(DC.FIRST, CrossDCTestEnricher::startAuthServerBackendNode); + forAllBackendNodesInDc(DC.SECOND, CrossDCTestEnricher::stopAuthServerBackendNode); + break; + } + + suspendPeriodicTasks(); + } + + public void afterTest(@Observes After event) { + if (!suiteContext.isAuthServerCrossDc()) return; + + restorePeriodicTasks(); + } + + public void stopSuiteContainers(@Observes(precedence = 4) StopSuiteContainers event) { + if (!suiteContext.isAuthServerCrossDc()) return; + + DC.validDcsStream().forEach(CrossDCTestEnricher::stopCacheServer); + forAllBackendNodes(CrossDCTestEnricher::stopAuthServerBackendNode); + } + + private static void createRESTClientsForNode(ContainerInfo node) { + if (!backendAdminClients.containsKey(node)) { + backendAdminClients.put(node, createAdminClientFor(node)); + } + + if (!backendTestingClients.containsKey(node)) { + backendTestingClients.put(node, createTestingClientFor(node)); + } + } + + private static void removeRESTClientsForNode(ContainerInfo node) { + if (backendAdminClients.containsKey(node)) { + backendAdminClients.get(node).close(); + backendAdminClients.remove(node); + } + + if (backendTestingClients.containsKey(node)) { + backendTestingClients.get(node).close(); + backendTestingClients.remove(node); + } + } + + public static Map getBackendAdminClients() { + return Collections.unmodifiableMap(backendAdminClients); + } + + public static Map getBackendTestingClients() { + return Collections.unmodifiableMap(backendTestingClients); + } + + private static Keycloak createAdminClientFor(ContainerInfo node) { + log.info("--DC: Initializing admin client for " + node.getContextRoot() + "/auth"); + return Keycloak.getInstance(node.getContextRoot() + "/auth", AuthRealm.MASTER, AuthRealm.ADMIN, AuthRealm.ADMIN, Constants.ADMIN_CLI_CLIENT_ID); + } + + private static KeycloakTestingClient createTestingClientFor(ContainerInfo node) { + log.info("--DC: Initializing testing client for " + node.getContextRoot() + "/auth"); + return KeycloakTestingClient.getInstance(node.getContextRoot() + "/auth"); + } + + // Disable periodic tasks in cross-dc tests. It's needed to have some scenarios more stable. + private static void suspendPeriodicTasks() { + log.debug("--DC: suspendPeriodicTasks"); + backendTestingClients.values().stream().forEach((KeycloakTestingClient testingClient) -> { + testingClient.testing().suspendPeriodicTasks(); + }); + } + + private static void restorePeriodicTasks() { + log.debug("--DC: restorePeriodicTasks"); + backendTestingClients.values().stream().forEach((KeycloakTestingClient testingClient) -> { + testingClient.testing().restorePeriodicTasks(); + }); + } + + /** + * Returns cache server corresponding to given DC + * @param dc + * @return + */ + private static ContainerInfo getCacheServer(DC dc) { + assertValidDc(dc); + int dcIndex = dc.ordinal(); + return suiteContext.getCacheServersInfo().get(dcIndex); + } + + private static void assertValidDc(DC dc) throws IllegalStateException { + if (dc == DC.UNDEFINED) { + throw new IllegalStateException("Invalid DC used: " + DC.UNDEFINED); + } + } + + public static void startCacheServer(DC dc) { + if (!containerController.get().isStarted(getCacheServer(dc).getQualifier())) { + log.infof("--DC: Starting %s", getCacheServer(dc).getQualifier()); + containerController.get().start(getCacheServer(dc).getQualifier()); + log.infof("--DC: Started %s", getCacheServer(dc).getQualifier()); + } + } + + public static void stopCacheServer(DC dc) { + String qualifier = getCacheServer(dc).getQualifier(); + + if (containerController.get().isStarted(qualifier)) { + log.infof("--DC: Stopping %s", qualifier); + + containerController.get().stop(qualifier); + + // Workaround for possible arquillian bug. Needs to cleanup dir manually + String setupCleanServerBaseDir = getContainerProperty(getCacheServer(dc), "setupCleanServerBaseDir"); + String cleanServerBaseDir = getContainerProperty(getCacheServer(dc), "cleanServerBaseDir"); + + if (Boolean.parseBoolean(setupCleanServerBaseDir)) { + log.debugf("Going to clean directory: %s", cleanServerBaseDir); + + File dir = new File(cleanServerBaseDir); + if (dir.exists()) { + try { + dir.renameTo(new File(dir.getParentFile(), dir.getName() + "-backup-" + System.currentTimeMillis())); + + File deploymentsDir = new File(dir, "deployments"); + FileUtils.forceMkdir(deploymentsDir); + } catch (IOException ioe) { + throw new RuntimeException("Failed to clean directory: " + cleanServerBaseDir, ioe); + } + } + } + + log.infof("--DC: Stopped %s", qualifier); + } + } + + public static void forAllBackendNodes(Consumer functionOnContainerInfo) { + suiteContext.getDcAuthServerBackendsInfo().stream() + .flatMap(Collection::stream) + .forEach(functionOnContainerInfo); + } + + public static void forAllBackendNodesInDc(DC dc, Consumer functionOnContainerInfo) { + assertValidDc(dc); + suiteContext.getDcAuthServerBackendsInfo().get(dc.ordinal()).stream() + .forEach(functionOnContainerInfo); + } + + public static void stopAuthServerBackendNode(ContainerInfo containerInfo) { + if (containerInfo.isStarted()) { + log.infof("--DC: Stopping backend auth-server node: %s", containerInfo.getQualifier()); + removeRESTClientsForNode(containerInfo); + containerController.get().stop(containerInfo.getQualifier()); + } + } + + public static void startAuthServerBackendNode(ContainerInfo containerInfo) { + if (! containerInfo.isStarted()) { + log.infof("--DC: Starting backend auth-server node: %s", containerInfo.getQualifier()); + containerController.get().start(containerInfo.getQualifier()); + createRESTClientsForNode(containerInfo); + } + } + + public static ContainerInfo getBackendNode(DC dc, int nodeIndex) { + assertValidDc(dc); + int dcIndex = dc.ordinal(); + assertThat((Integer) dcIndex, lessThan(suiteContext.getDcAuthServerBackendsInfo().size())); + final List dcNodes = suiteContext.getDcAuthServerBackendsInfo().get(dcIndex); + assertThat((Integer) nodeIndex, lessThan(dcNodes.size())); + return dcNodes.get(nodeIndex); + } + + /** + * Starts a manually-controlled backend auth-server node in cross-DC scenario. + * @param dc + * @param nodeIndex + * @return Started instance descriptor. + */ + public static ContainerInfo startAuthServerBackendNode(DC dc, int nodeIndex) { + ContainerInfo dcNode = getBackendNode(dc, nodeIndex); + startAuthServerBackendNode(dcNode); + return dcNode; + } + + /** + * Stops a manually-controlled backend auth-server node in cross-DC scenario. + * @param dc + * @param nodeIndex + * @return Stopped instance descriptor. + */ + public static ContainerInfo stopAuthServerBackendNode(DC dc, int nodeIndex) { + ContainerInfo dcNode = getBackendNode(dc, nodeIndex); + stopAuthServerBackendNode(dcNode); + return dcNode; + } + + private Class getNearestSuperclassWithAnnotation(Class testClass, Class annotationClass) { + return (testClass.isAnnotationPresent(annotationClass)) ? testClass + : (testClass.getSuperclass().equals(Object.class) ? null // stop recursion + : getNearestSuperclassWithAnnotation(testClass.getSuperclass(), annotationClass)); // continue recursion + } + + private static String getContainerProperty(ContainerInfo cacheServer, String propertyName) { + return cacheServer.getArquillianContainer().getContainerConfiguration().getContainerProperties().get(propertyName); + } +} diff --git a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/KeycloakArquillianExtension.java b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/KeycloakArquillianExtension.java index b3b4e53653..eb6eb01b57 100644 --- a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/KeycloakArquillianExtension.java +++ b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/KeycloakArquillianExtension.java @@ -66,7 +66,7 @@ public class KeycloakArquillianExtension implements LoadableExtension { .observer(JmxConnectorRegistryCreator.class) .observer(AuthServerTestEnricher.class) .observer(AppServerTestEnricher.class) - .observer(CacheServerTestEnricher.class) + .observer(CrossDCTestEnricher.class) .observer(H2TestEnricher.class); builder .service(TestExecutionDecider.class, MigrationTestExecutionDecider.class) diff --git a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/annotation/InitialDcState.java b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/annotation/InitialDcState.java new file mode 100644 index 0000000000..49e177a766 --- /dev/null +++ b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/annotation/InitialDcState.java @@ -0,0 +1,44 @@ +/* + * Copyright 201 Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.keycloak.testsuite.arquillian.annotation; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import org.keycloak.testsuite.crossdc.ServerSetup; + +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +/** + * Specifies initial state of auth-server and cache-server nodes before start of each test + * in multinode setup like in cross-DC tests. + * When a test class is annotated, this annotation is applied to every test method in the class + * but can be overridden on method level. + * + * @author vramik + * @author hmlnarik + */ +@Documented +@Retention(RUNTIME) +@Target({ElementType.TYPE, ElementType.METHOD}) +public @interface InitialDcState { + ServerSetup cacheServers() default ServerSetup.FIRST_NODE_IN_EVERY_DC; + ServerSetup authServers() default ServerSetup.FIRST_NODE_IN_EVERY_DC; +} diff --git a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/crossdc/DC.java b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/crossdc/DC.java index 1ed8cad64c..6460742bec 100644 --- a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/crossdc/DC.java +++ b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/crossdc/DC.java @@ -16,6 +16,9 @@ */ package org.keycloak.testsuite.crossdc; +import java.util.Arrays; +import java.util.stream.Stream; + /** * Identifier of datacentre in the testsuite * @author hmlnarik @@ -28,4 +31,10 @@ public enum DC { public int getDcIndex() { return ordinal(); } + + private static final DC[] VALID_DCS = new DC[] { FIRST, SECOND }; + + public static Stream validDcsStream() { + return Arrays.stream(VALID_DCS); + } } diff --git a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/crossdc/ServerSetup.java b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/crossdc/ServerSetup.java new file mode 100644 index 0000000000..6e71b08d12 --- /dev/null +++ b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/crossdc/ServerSetup.java @@ -0,0 +1,28 @@ +/* + * Copyright 2018 Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.keycloak.testsuite.crossdc; + +/** + * @author vramik + */ +public enum ServerSetup { + FIRST_NODE_IN_FIRST_DC, + FIRST_NODE_IN_EVERY_DC, + ALL_NODES_IN_EVERY_DC, + ALL_NODES_IN_FIRST_DC_FIRST_NODE_IN_SECOND_DC, + ALL_NODES_IN_FIRST_DC_NO_NODES_IN_SECOND_DC; +} diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractCrossDCTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractCrossDCTest.java index c4d48574d1..6bd25fee58 100644 --- a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractCrossDCTest.java +++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractCrossDCTest.java @@ -16,38 +16,31 @@ */ package org.keycloak.testsuite.crossdc; -import org.apache.commons.io.FileUtils; import org.keycloak.admin.client.Keycloak; -import org.keycloak.models.Constants; import org.keycloak.testsuite.AbstractTestRealmKeycloakTest; import org.keycloak.testsuite.arquillian.AuthServerTestEnricher; import org.keycloak.testsuite.arquillian.ContainerInfo; import org.keycloak.testsuite.arquillian.LoadBalancerController; import org.keycloak.testsuite.arquillian.annotation.LoadBalancer; -import org.keycloak.testsuite.auth.page.AuthRealm; -import java.io.File; -import java.io.IOException; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; import org.jboss.arquillian.container.test.api.ContainerController; import org.jboss.arquillian.test.api.ArquillianResource; import org.junit.After; import org.junit.Before; import org.keycloak.testsuite.client.KeycloakTestingClient; -import static org.hamcrest.Matchers.lessThan; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; import org.keycloak.representations.idm.RealmRepresentation; +import org.keycloak.testsuite.arquillian.CrossDCTestEnricher; +import org.keycloak.testsuite.arquillian.annotation.InitialDcState; /** * Abstract cross-data-centre test that defines primitives for handling cross-DC setup. * @author hmlnarik */ +@InitialDcState public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest { // Keep the following constants in sync with arquillian @@ -59,59 +52,34 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest @LoadBalancer(value = QUALIFIER_NODE_BALANCER) protected LoadBalancerController loadBalancerCtrl; - @ArquillianResource - protected ContainerController containerController; - - protected Map backendAdminClients = new HashMap<>(); - - protected Map backendTestingClients = new HashMap<>(); - @Before @Override public void beforeAbstractKeycloakTest() throws Exception { - log.debug("--DC: Starting both cache servers and first node from each DC"); - startCacheServer(DC.FIRST); - startCacheServer(DC.SECOND); - - startBackendNode(DC.FIRST, 0); - startBackendNode(DC.SECOND, 0); - - initRESTClientsForStartedNodes(); - suspendPeriodicTasks(); - enableOnlyFirstNodeInFirstDc(); super.beforeAbstractKeycloakTest(); } - @Override - public void deleteCookies() { - //Overrides AbstractTestRealmKeycloakTest.deleteCookies - //as it tries to delete cookies in 'test' realm @After test - //when backend containers are stopped already. - } - @After @Override public void afterAbstractKeycloakTest() { log.debug("--DC: after AbstractCrossDCTest"); + CrossDCTestEnricher.startAuthServerBackendNode(DC.FIRST, 0); // make sure first node is started enableOnlyFirstNodeInFirstDc(); - + super.afterAbstractKeycloakTest(); - - restorePeriodicTasks(); + removeTestRealms(); - terminateStartedServers(); loadBalancerCtrl.disableAllBackendNodes(); } private void enableOnlyFirstNodeInFirstDc() { log.debug("--DC: Enable only first node in first datacenter"); this.loadBalancerCtrl.disableAllBackendNodes(); - if (!getBackendNode(DC.FIRST, 0).isStarted()) { + if (!CrossDCTestEnricher.getBackendNode(DC.FIRST, 0).isStarted()) { throw new IllegalStateException("--DC: Trying to enable not started node on load-balancer"); } - loadBalancerCtrl.enableBackendNodeByName(getBackendNode(DC.FIRST, 0).getQualifier()); + loadBalancerCtrl.enableBackendNodeByName(CrossDCTestEnricher.getBackendNode(DC.FIRST, 0).getQualifier()); } private void removeTestRealms() { @@ -121,52 +89,6 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest AuthServerTestEnricher.removeTestRealms(testContext, adminClient); testContext.setTestRealmReps(new ArrayList<>()); } - - protected void terminateStartedServers() { - log.debug("--DC: Halting all nodes that are started"); - this.suiteContext.getDcAuthServerBackendsInfo().stream() - .flatMap(List::stream) - .filter(ContainerInfo::isStarted) - .forEach((ContainerInfo containerInfo) -> { - containerController.stop(containerInfo.getQualifier()); - removeRESTClientsForNode(containerInfo); - }); - } - - private void initRESTClientsForStartedNodes() { - log.debug("--DC: Init REST clients for started nodes"); - this.suiteContext.getDcAuthServerBackendsInfo().stream() - .flatMap(List::stream) - .filter(ContainerInfo::isStarted) - .forEach(containerInfo -> { - createRESTClientsForNode(containerInfo); - }); - } - - // Disable periodic tasks in cross-dc tests. It's needed to have some scenarios more stable. - private void suspendPeriodicTasks() { - log.debug("--DC: suspendPeriodicTasks"); - backendTestingClients.values().stream().forEach((KeycloakTestingClient testingClient) -> { - testingClient.testing().suspendPeriodicTasks(); - }); - } - - private void restorePeriodicTasks() { - log.debug("--DC: restorePeriodicTasks"); - backendTestingClients.values().stream().forEach((KeycloakTestingClient testingClient) -> { - testingClient.testing().restorePeriodicTasks(); - }); - } - - protected Keycloak createAdminClientFor(ContainerInfo node) { - log.info("--DC: Initializing admin client for " + node.getContextRoot() + "/auth"); - return Keycloak.getInstance(node.getContextRoot() + "/auth", AuthRealm.MASTER, AuthRealm.ADMIN, AuthRealm.ADMIN, Constants.ADMIN_CLI_CLIENT_ID); - } - - protected KeycloakTestingClient createTestingClientFor(ContainerInfo node) { - log.info("--DC: Initializing testing client for " + node.getContextRoot() + "/auth"); - return KeycloakTestingClient.getInstance(node.getContextRoot() + "/auth"); - } protected Keycloak getAdminClientForStartedNodeInDc(int dcIndex) { ContainerInfo firstStartedNode = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex).stream() @@ -182,14 +104,13 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest * @return */ protected Keycloak getAdminClientFor(ContainerInfo node) { - Keycloak client = backendAdminClients.get(node); + Keycloak client = CrossDCTestEnricher.getBackendAdminClients().get(node); if (client == null && node.equals(suiteContext.getAuthServerInfo())) { client = this.adminClient; } return client; } - protected KeycloakTestingClient getTestingClientForStartedNodeInDc(int dcIndex) { ContainerInfo firstStartedNode = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex).stream() .filter(ContainerInfo::isStarted) @@ -204,35 +125,13 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest * @return */ protected KeycloakTestingClient getTestingClientFor(ContainerInfo node) { - KeycloakTestingClient client = backendTestingClients.get(node); + KeycloakTestingClient client = CrossDCTestEnricher.getBackendTestingClients().get(node); if (client == null && node.equals(suiteContext.getAuthServerInfo())) { client = this.testingClient; } return client; } - protected void createRESTClientsForNode(ContainerInfo node) { - if (!backendAdminClients.containsKey(node)) { - backendAdminClients.put(node, createAdminClientFor(node)); - } - - if (!backendTestingClients.containsKey(node)) { - backendTestingClients.put(node, createTestingClientFor(node)); - } - } - - protected void removeRESTClientsForNode(ContainerInfo node) { - if (backendAdminClients.containsKey(node)) { - backendAdminClients.get(node).close(); - backendAdminClients.remove(node); - } - - if (backendTestingClients.containsKey(node)) { - backendTestingClients.get(node).close(); - backendTestingClients.remove(node); - } - } - /** * Disables routing requests to the given data center in the load balancer. * @param dc @@ -293,97 +192,6 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest loadBalancerCtrl.enableBackendNodeByName(backendNode.getQualifier()); } - /** - * Starts a manually-controlled backend auth-server node in cross-DC scenario. - * @param dc - * @param nodeIndex - * @return Started instance descriptor. - */ - protected ContainerInfo startBackendNode(DC dc, int nodeIndex) { - ContainerInfo dcNode = getBackendNode(dc, nodeIndex); - - assertTrue("Node " + dcNode.getQualifier() + " has to be controlled manually", dcNode.isManual()); - - if (!containerController.isStarted(dcNode.getQualifier())) { - log.infof("--DC: Starting backend node: %s (dcIndex: %d, nodeIndex: %d)", dcNode.getQualifier(), dc.ordinal(), nodeIndex); - containerController.start(dcNode.getQualifier()); - - createRESTClientsForNode(dcNode); - } - return dcNode; - } - - /** - * Stops a manually-controlled backend auth-server node in cross-DC scenario. - * @param dc - * @param nodeIndex - * @return Stopped instance descriptor. - */ - protected ContainerInfo stopBackendNode(DC dc, int nodeIndex) { - ContainerInfo dcNode = getBackendNode(dc, nodeIndex); - - removeRESTClientsForNode(dcNode); - - assertTrue("Node " + dcNode.getQualifier() + " has to be controlled manually", dcNode.isManual()); - - log.infof("--DC: Stopping backend node: %s (dcIndex: %d, nodeIndex: %d)", dcNode.getQualifier(), dc.ordinal(), nodeIndex); - containerController.stop(dcNode.getQualifier()); - return dcNode; - } - - private ContainerInfo getBackendNode(DC dc, int nodeIndex) { - int dcIndex = dc.ordinal(); - assertThat((Integer) dcIndex, lessThan(this.suiteContext.getDcAuthServerBackendsInfo().size())); - final List dcNodes = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex); - assertThat((Integer) nodeIndex, lessThan(dcNodes.size())); - return dcNodes.get(nodeIndex); - } - - /** - * Returns cache server corresponding to given DC - * @param dc - * @return - */ - protected ContainerInfo getCacheServer(DC dc) { - int dcIndex = dc.ordinal(); - return this.suiteContext.getCacheServersInfo().get(dcIndex); - } - - protected void startCacheServer(DC dc) { - if (!containerController.isStarted(getCacheServer(dc).getQualifier())) { - log.infof("--DC: Starting %s", getCacheServer(dc).getQualifier()); - containerController.start(getCacheServer(dc).getQualifier()); - } - } - - protected void stopCacheServer(ContainerInfo cacheServer) { - log.infof("--DC: Stopping %s", cacheServer.getQualifier()); - - containerController.stop(cacheServer.getQualifier()); - - // Workaround for possible arquillian bug. Needs to cleanup dir manually - String setupCleanServerBaseDir = cacheServer.getArquillianContainer().getContainerConfiguration().getContainerProperties().get("setupCleanServerBaseDir"); - String cleanServerBaseDir = cacheServer.getArquillianContainer().getContainerConfiguration().getContainerProperties().get("cleanServerBaseDir"); - - if (Boolean.parseBoolean(setupCleanServerBaseDir)) { - log.infof("--DC: Going to clean directory: %s", cleanServerBaseDir); - - File dir = new File(cleanServerBaseDir); - if (dir.exists()) { - try { - dir.renameTo(new File(dir.getParentFile(), dir.getName() + "--" + System.currentTimeMillis())); - - File deploymentsDir = new File(dir, "deployments"); - FileUtils.forceMkdir(deploymentsDir); - } catch (IOException ioe) { - throw new RuntimeException("Failed to clean directory: " + cleanServerBaseDir, ioe); - } - } - } - - log.infof("--DC: Stopped %s", cacheServer.getQualifier()); - } - /** * Sets time offset on all the started containers. * @@ -396,7 +204,7 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest } private void setTimeOffsetOnAllStartedContainers(int offset) { - backendTestingClients.entrySet().stream() + CrossDCTestEnricher.getBackendTestingClients().entrySet().stream() .filter(testingClientEntry -> testingClientEntry.getKey().isStarted()) .map(testingClientEntry -> testingClientEntry.getValue()) .forEach(testingClient -> testingClient.testing().setTimeOffset(Collections.singletonMap("offset", String.valueOf(offset)))); diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/ActionTokenCrossDCTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/ActionTokenCrossDCTest.java index bb5d341eb1..5a866651a7 100644 --- a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/ActionTokenCrossDCTest.java +++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/ActionTokenCrossDCTest.java @@ -50,6 +50,8 @@ import org.hamcrest.Matchers; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.greaterThan; import static org.junit.Assert.assertThat; +import org.keycloak.testsuite.arquillian.CrossDCTestEnricher; +import org.keycloak.testsuite.arquillian.annotation.InitialDcState; /** * @@ -80,6 +82,7 @@ public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest { } @Test + @InitialDcState(authServers = ServerSetup.ALL_NODES_IN_FIRST_DC_FIRST_NODE_IN_SECOND_DC) public void sendResetPasswordEmailSuccessWorksInCrossDc( @JmxInfinispanCacheStatistics(dc=DC.FIRST, dcNodeIndex=0, cacheName=InfinispanConnectionProvider.ACTION_TOKEN_CACHE) InfinispanStatistics cacheDc0Node0Statistics, @JmxInfinispanCacheStatistics(dc=DC.FIRST, dcNodeIndex=1, cacheName=InfinispanConnectionProvider.ACTION_TOKEN_CACHE) InfinispanStatistics cacheDc0Node1Statistics, @@ -87,7 +90,6 @@ public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest { @JmxInfinispanChannelStatistics() InfinispanStatistics channelStatisticsCrossDc) throws Exception { log.debug("--DC: START sendResetPasswordEmailSuccessWorksInCrossDc"); - startBackendNode(DC.FIRST, 1); cacheDc0Node1Statistics.waitToBecomeAvailable(10, TimeUnit.SECONDS); Comparable originalNumberOfEntries = cacheDc0Node0Statistics.getSingleStatistics(Constants.STAT_CACHE_NUMBER_OF_ENTRIES); @@ -155,6 +157,7 @@ public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest { } @Test + @InitialDcState(authServers = ServerSetup.FIRST_NODE_IN_FIRST_DC) public void sendResetPasswordEmailAfterNewNodeAdded() throws IOException, MessagingException { log.debug("--DC: START sendResetPasswordEmailAfterNewNodeAdded"); disableDcOnLoadBalancer(DC.SECOND); @@ -188,7 +191,8 @@ public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest { assertEquals("Your account has been updated.", PageUtils.getPageTitle(driver)); disableDcOnLoadBalancer(DC.FIRST); - startBackendNode(DC.SECOND, 1); + CrossDCTestEnricher.startAuthServerBackendNode(DC.SECOND, 1); + CrossDCTestEnricher.stopAuthServerBackendNode(DC.FIRST, 0); enableLoadBalancerNode(DC.SECOND, 1); Retry.execute(() -> { diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/ConcurrentLoginCrossDCTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/ConcurrentLoginCrossDCTest.java index 99f8ee3af5..33deb9b8a3 100644 --- a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/ConcurrentLoginCrossDCTest.java +++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/ConcurrentLoginCrossDCTest.java @@ -37,10 +37,12 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.LaxRedirectStrategy; import org.junit.Test; +import org.keycloak.testsuite.arquillian.annotation.InitialDcState; /** * @author Marek Posolda */ +@InitialDcState(authServers = ServerSetup.ALL_NODES_IN_EVERY_DC) public class ConcurrentLoginCrossDCTest extends ConcurrentLoginTest { @ArquillianResource @@ -56,35 +58,11 @@ public class ConcurrentLoginCrossDCTest extends ConcurrentLoginTest { @Override public void beforeAbstractKeycloakTestRealmImport() { - log.debug("--DC: Starting cacheServers if not started already"); - suiteContext.getCacheServersInfo().stream() - .filter((containerInfo) -> !containerInfo.isStarted()) - .map(ContainerInfo::getQualifier) - .forEach(containerController::start); - - log.debug("--DC: Initializing load balancer - enabling all started nodes across DCs"); - this.loadBalancerCtrl.disableAllBackendNodes(); - - this.suiteContext.getDcAuthServerBackendsInfo().stream() - .flatMap(List::stream) - .filter((containerInfo) -> !containerInfo.getQualifier().contains("manual")) - .filter((containerInfo) -> !containerInfo.isStarted()) - .map(ContainerInfo::getQualifier) - .forEach((nodeName) -> { - containerController.start(nodeName); - loadBalancerCtrl.enableBackendNodeByName(nodeName); - }); + loadBalancerCtrl.enableAllBackendNodes(); } @Override public void postAfterAbstractKeycloak() { - log.debug("--DC: postAfterAbstractKeycloak"); - suiteContext.getDcAuthServerBackendsInfo().stream() - .flatMap(List::stream) - .filter(ContainerInfo::isStarted) - .map(ContainerInfo::getQualifier) - .forEach(containerController::stop); - loadBalancerCtrl.disableAllBackendNodes(); //realms is already removed and this prevents another removal in AuthServerTestEnricher.afterClass diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/SessionExpirationCrossDCTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/SessionExpirationCrossDCTest.java index c32ab7b386..643bb4b674 100644 --- a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/SessionExpirationCrossDCTest.java +++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/SessionExpirationCrossDCTest.java @@ -40,7 +40,9 @@ import org.keycloak.representations.idm.UserSessionRepresentation; import org.keycloak.testsuite.Assert; import org.keycloak.common.util.Retry; import org.keycloak.testsuite.admin.ApiUtil; +import org.keycloak.testsuite.arquillian.CrossDCTestEnricher; import org.keycloak.testsuite.arquillian.InfinispanStatistics; +import org.keycloak.testsuite.arquillian.annotation.InitialDcState; import org.keycloak.testsuite.arquillian.annotation.JmxInfinispanCacheStatistics; import org.keycloak.testsuite.arquillian.annotation.JmxInfinispanChannelStatistics; import org.keycloak.testsuite.util.ClientBuilder; @@ -417,20 +419,18 @@ public class SessionExpirationCrossDCTest extends AbstractAdminCrossDCTest { @Test + @InitialDcState(authServers = ServerSetup.ALL_NODES_IN_FIRST_DC_FIRST_NODE_IN_SECOND_DC) public void testLogoutUserWithFailover( @JmxInfinispanCacheStatistics(dc=DC.FIRST, managementPortProperty = "cache.server.management.port", cacheName=InfinispanConnectionProvider.USER_SESSION_CACHE_NAME) InfinispanStatistics cacheDc1Statistics, @JmxInfinispanCacheStatistics(dc=DC.SECOND, managementPortProperty = "cache.server.2.management.port", cacheName=InfinispanConnectionProvider.USER_SESSION_CACHE_NAME) InfinispanStatistics cacheDc2Statistics, @JmxInfinispanChannelStatistics() InfinispanStatistics channelStatisticsCrossDc) throws Exception { - // Start node2 on first DC - startBackendNode(DC.FIRST, 1); - // Don't include remote stats. Size is smaller because of distributed cache List responses = createInitialSessions(InfinispanConnectionProvider.USER_SESSION_CACHE_NAME, InfinispanConnectionProvider.CLIENT_SESSION_CACHE_NAME, false, cacheDc1Statistics, cacheDc2Statistics, false); // Kill node2 now. Around 10 sessions (half of SESSIONS_COUNT) will be lost on Keycloak side. But not on infinispan side - stopBackendNode(DC.FIRST, 1); + CrossDCTestEnricher.stopAuthServerBackendNode(DC.FIRST, 1); channelStatisticsCrossDc.reset(); @@ -461,15 +461,12 @@ public class SessionExpirationCrossDCTest extends AbstractAdminCrossDCTest { @Test + @InitialDcState(authServers = ServerSetup.ALL_NODES_IN_EVERY_DC) public void testLogoutWithAllStartedNodes( @JmxInfinispanCacheStatistics(dc=DC.FIRST, managementPortProperty = "cache.server.management.port", cacheName=InfinispanConnectionProvider.USER_SESSION_CACHE_NAME) InfinispanStatistics cacheDc1Statistics, @JmxInfinispanCacheStatistics(dc=DC.SECOND, managementPortProperty = "cache.server.2.management.port", cacheName=InfinispanConnectionProvider.USER_SESSION_CACHE_NAME) InfinispanStatistics cacheDc2Statistics, @JmxInfinispanChannelStatistics() InfinispanStatistics channelStatisticsCrossDc) throws Exception { - // Start node2 on every DC - startBackendNode(DC.FIRST, 1); - startBackendNode(DC.SECOND, 1); - // Create sessions. Don't include remote stats. Size is smaller because of distributed cache List responses = createInitialSessions(InfinispanConnectionProvider.USER_SESSION_CACHE_NAME, InfinispanConnectionProvider.CLIENT_SESSION_CACHE_NAME, false, cacheDc1Statistics, cacheDc2Statistics, false); @@ -505,8 +502,8 @@ public class SessionExpirationCrossDCTest extends AbstractAdminCrossDCTest { }, 50, 50); // Stop both nodes - stopBackendNode(DC.FIRST, 1); - stopBackendNode(DC.SECOND, 1); + CrossDCTestEnricher.stopAuthServerBackendNode(DC.FIRST, 1); + CrossDCTestEnricher.stopAuthServerBackendNode(DC.SECOND, 1); } private void assertTestAppActiveSessionsCount(int expectedSessionsCount) { diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/SessionsPreloadCrossDCTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/SessionsPreloadCrossDCTest.java index 17453c0a9b..a39736d3b7 100644 --- a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/SessionsPreloadCrossDCTest.java +++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/SessionsPreloadCrossDCTest.java @@ -23,10 +23,13 @@ import java.util.List; import org.junit.Before; import org.junit.Test; import org.keycloak.OAuth2Constants; +import org.keycloak.common.util.Retry; import org.keycloak.connections.infinispan.InfinispanConnectionProvider; import org.keycloak.representations.idm.RealmRepresentation; import org.keycloak.testsuite.Assert; import org.keycloak.testsuite.admin.ApiUtil; +import org.keycloak.testsuite.arquillian.CrossDCTestEnricher; +import org.keycloak.testsuite.arquillian.annotation.InitialDcState; import org.keycloak.testsuite.util.OAuthClient; /** @@ -34,38 +37,28 @@ import org.keycloak.testsuite.util.OAuthClient; * * @author Marek Posolda */ +@InitialDcState(authServers = ServerSetup.ALL_NODES_IN_FIRST_DC_NO_NODES_IN_SECOND_DC) public class SessionsPreloadCrossDCTest extends AbstractAdminCrossDCTest { private static final int SESSIONS_COUNT = 10; @Before public void beforeSessionsPreloadCrossDCTest() throws Exception { - // Start DC1 and only All Keycloak nodes on DC2 are stopped - stopBackendNode(DC.SECOND, 0); disableDcOnLoadBalancer(DC.SECOND); } - private void stopAllCacheServersAndAuthServers() { log.infof("Going to stop all auth servers"); - stopBackendNode(DC.FIRST, 0); - disableLoadBalancerNode(DC.FIRST, 0); - stopBackendNode(DC.SECOND, 0); - disableLoadBalancerNode(DC.SECOND, 0); + CrossDCTestEnricher.forAllBackendNodes(CrossDCTestEnricher::stopAuthServerBackendNode); + loadBalancerCtrl.disableAllBackendNodes(); log.infof("Auth servers stopped successfully. Going to stop all cache servers"); - suiteContext.getCacheServersInfo().stream() - .filter(containerInfo -> containerInfo.isStarted()) - .forEach(containerInfo -> { - stopCacheServer(containerInfo); - }); - + DC.validDcsStream().forEach(CrossDCTestEnricher::stopCacheServer); log.infof("Cache servers stopped successfully"); } - @Test public void sessionsPreloadTest() throws Exception { int sessionsBefore = getTestingClientForStartedNodeInDc(0).testing().cache(InfinispanConnectionProvider.USER_SESSION_CACHE_NAME).size(); @@ -75,7 +68,7 @@ public class SessionsPreloadCrossDCTest extends AbstractAdminCrossDCTest { List tokenResponses = createInitialSessions(false); // Start 2nd DC. - startBackendNode(DC.SECOND, 0); + CrossDCTestEnricher.startAuthServerBackendNode(DC.SECOND, 0); enableLoadBalancerNode(DC.SECOND, 0); // Ensure sessions are loaded in both 1st DC and 2nd DC @@ -113,15 +106,14 @@ public class SessionsPreloadCrossDCTest extends AbstractAdminCrossDCTest { stopAllCacheServersAndAuthServers(); // Start cache containers on both DC1 and DC2 - startCacheServer(DC.FIRST); - startCacheServer(DC.SECOND); + DC.validDcsStream().forEach(CrossDCTestEnricher::startCacheServer); // Start Keycloak on DC1. Sessions should be preloaded from DB - startBackendNode(DC.FIRST, 0); + CrossDCTestEnricher.startAuthServerBackendNode(DC.FIRST, 0); enableLoadBalancerNode(DC.FIRST, 0); // Start Keycloak on DC2. Sessions should be preloaded from remoteCache - startBackendNode(DC.SECOND, 0); + CrossDCTestEnricher.startAuthServerBackendNode(DC.SECOND, 0); enableLoadBalancerNode(DC.SECOND, 0); // Ensure sessions are loaded in both 1st DC and 2nd DC @@ -167,19 +159,21 @@ public class SessionsPreloadCrossDCTest extends AbstractAdminCrossDCTest { } // Start 2nd DC. - startBackendNode(DC.SECOND, 0); + CrossDCTestEnricher.startAuthServerBackendNode(DC.SECOND, 0); enableLoadBalancerNode(DC.SECOND, 0); - // Ensure loginFailures are loaded in both 1st DC and 2nd DC - int size1 = getTestingClientForStartedNodeInDc(0).testing().cache(InfinispanConnectionProvider.LOGIN_FAILURE_CACHE_NAME).size(); - int size2 = getTestingClientForStartedNodeInDc(1).testing().cache(InfinispanConnectionProvider.LOGIN_FAILURE_CACHE_NAME).size(); - int loginFailures1 = (Integer) getAdminClientForStartedNodeInDc(0).realm("test").attackDetection().bruteForceUserStatus(userId).get("numFailures"); - int loginFailures2 = (Integer) getAdminClientForStartedNodeInDc(1).realm("test").attackDetection().bruteForceUserStatus(userId).get("numFailures"); - log.infof("size1: %d, size2: %d, loginFailures1: %d, loginFailures2: %d", size1, size2, loginFailures1, loginFailures2); - Assert.assertEquals(size1, 1); - Assert.assertEquals(size2, 1); - Assert.assertEquals(loginFailures1, loginFailuresBefore + SESSIONS_COUNT); - Assert.assertEquals(loginFailures2, loginFailuresBefore + SESSIONS_COUNT); + Retry.execute(() -> { + // Ensure loginFailures are loaded in both 1st DC and 2nd DC + int size1 = getTestingClientForStartedNodeInDc(0).testing().cache(InfinispanConnectionProvider.LOGIN_FAILURE_CACHE_NAME).size(); + int size2 = getTestingClientForStartedNodeInDc(1).testing().cache(InfinispanConnectionProvider.LOGIN_FAILURE_CACHE_NAME).size(); + int loginFailures1 = (Integer) getAdminClientForStartedNodeInDc(0).realm("test").attackDetection().bruteForceUserStatus(userId).get("numFailures"); + int loginFailures2 = (Integer) getAdminClientForStartedNodeInDc(1).realm("test").attackDetection().bruteForceUserStatus(userId).get("numFailures"); + log.infof("size1: %d, size2: %d, loginFailures1: %d, loginFailures2: %d", size1, size2, loginFailures1, loginFailures2); + Assert.assertEquals(size1, 1); + Assert.assertEquals(size2, 1); + Assert.assertEquals(loginFailures1, loginFailuresBefore + SESSIONS_COUNT); + Assert.assertEquals(loginFailures2, loginFailuresBefore + SESSIONS_COUNT); + }, 3, 400); // On DC2 sessions were preloaded from from remoteCache Assert.assertTrue(getTestingClientForStartedNodeInDc(1).testing().cache(InfinispanConnectionProvider.WORK_CACHE_NAME).contains("distributed::remoteCacheLoad::loginFailures")); diff --git a/testsuite/integration-arquillian/tests/pom.xml b/testsuite/integration-arquillian/tests/pom.xml index 2a980324d4..5b5ad1f8f8 100755 --- a/testsuite/integration-arquillian/tests/pom.xml +++ b/testsuite/integration-arquillian/tests/pom.xml @@ -371,7 +371,7 @@ maven-antrun-plugin - clean-second-cache-server-arquillian-bug-workaround + clean-second-cache-server-arquillian-bug-workaround process-test-resources run diff --git a/travis-run-tests.sh b/travis-run-tests.sh index 9adbdd000d..c3e476026b 100755 --- a/travis-run-tests.sh +++ b/travis-run-tests.sh @@ -74,7 +74,7 @@ if [ $1 == "server-group4" ]; then run-server-tests org.keycloak.testsuite.k*.**.*Test,org.keycloak.testsuite.m*.**.*Test,org.keycloak.testsuite.o*.**.*Test,org.keycloak.testsuite.s*.**.*Test fi -if [ $1 == "crossdc1" ]; then +if [ $1 == "crossdc-server" ]; then cd testsuite/integration-arquillian mvn install -B -nsu -Pauth-servers-crossdc-jboss,auth-server-wildfly,cache-server-infinispan -DskipTests @@ -84,7 +84,7 @@ if [ $1 == "crossdc1" ]; then exit ${PIPESTATUS[0]} fi -if [ $1 == "crossdc2" ]; then +if [ $1 == "crossdc-adapter" ]; then cd testsuite/integration-arquillian mvn install -B -nsu -Pauth-servers-crossdc-jboss,auth-server-wildfly,cache-server-infinispan,app-server-wildfly -DskipTests