diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be19c95767..1c667ce48c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -185,7 +185,7 @@ jobs: PARAMS["sanity-check-zip"]="-Dtest=StartCommandDistTest,StartDevCommandDistTest,BuildAndStartDistTest,ImportAtStartupDistTest" PARAMS["zip"]="" PARAMS["container"]="-Dkc.quarkus.tests.dist=docker" - PARAMS["storage"]="-Ptest-database -Dtest=PostgreSQLDistTest,MariaDBDistTest#testSuccessful,MySQLDistTest#testSuccessful,DatabaseOptionsDistTest,JPAStoreDistTest,HotRodStoreDistTest,MixedStoreDistTest,TransactionConfigurationDistTest" + PARAMS["storage"]="-Ptest-database -Dtest=PostgreSQLDistTest,MariaDBDistTest#testSuccessful,MySQLDistTest#testSuccessful,DatabaseOptionsDistTest,JPAStoreDistTest,HotRodStoreDistTest,MixedStoreDistTest,TransactionConfigurationDistTest,ExternalInfinispanTest" ./mvnw install -pl quarkus/tests/integration -am -DskipTests ./mvnw test -pl quarkus/tests/integration ${PARAMS["${{ matrix.server }}"]} | misc/log/trimmer.sh diff --git a/docs/guides/high-availability/concepts-active-passive-sync.adoc b/docs/guides/high-availability/concepts-active-passive-sync.adoc index aa955f3258..726814102e 100644 --- a/docs/guides/high-availability/concepts-active-passive-sync.adoc +++ b/docs/guides/high-availability/concepts-active-passive-sync.adoc @@ -62,13 +62,11 @@ Monitoring is necessary to detect degraded setups. | {jdgserver_name} cluster failure | If the {jdgserver_name} cluster fails in the active site, {project_name} will not be able to communicate with the external {jdgserver_name}, and the {project_name} service will be unavailable. -Manual switchover to the secondary site is recommended. -Future versions will detect this situation and do an automatic failover. +The loadbalancer will detect the situation as `/lb-check` returns an error, and will fail over to the other site. -When the {jdgserver_name} cluster is restored, its data will be out-of-sync with {project_name}. -Manual operations are required to get {jdgserver_name} in the primary site in sync with the secondary site. -| Loss of service -| Human intervention required +The setup is degraded until the {jdgserver_name} cluster is restored and the session data is re-synchronized to the primary. +| No data loss^3^ +| Seconds to minutes (depending on load balancer setup) | Connectivity {jdgserver_name} | If the connectivity between the two sites is lost, session information cannot be sent to the other site. diff --git a/quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/HealthDistTest.java b/quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/HealthDistTest.java index 7c359870ae..20134a4a3e 100644 --- a/quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/HealthDistTest.java +++ b/quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/HealthDistTest.java @@ -47,6 +47,8 @@ public class HealthDistTest { .statusCode(404); when().get("/q/health/ready").then() .statusCode(404); + when().get("/lb-check").then() + .statusCode(404); } @Test @@ -61,6 +63,8 @@ public class HealthDistTest { // Metrics should not be enabled when().get("/metrics").then() .statusCode(404); + when().get("/lb-check").then() + .statusCode(404); } @Test @@ -72,6 +76,8 @@ public class HealthDistTest { .statusCode(200) .body("checks[0].name", equalTo("Keycloak database connections async health check")) .body("checks.size()", equalTo(1)); + when().get("/lb-check").then() + .statusCode(404); } @Test @@ -83,6 +89,8 @@ public class HealthDistTest { .statusCode(200) .body("checks[0].name", equalTo("Keycloak database connections health check")) .body("checks.size()", equalTo(1)); + when().get("/lb-check").then() + .statusCode(404); } @Test @@ -125,4 +133,11 @@ public class HealthDistTest { distribution.stop(); } } + + @Test + @Launch({ "start-dev", "--features=multi-site" }) + void testLoadBalancerCheck() { + when().get("/lb-check").then() + .statusCode(200); + } } diff --git a/quarkus/tests/integration/src/test/java/org/keycloak/it/storage/database/ExternalInfinispanTest.java b/quarkus/tests/integration/src/test/java/org/keycloak/it/storage/database/ExternalInfinispanTest.java new file mode 100644 index 0000000000..c13cd89703 --- /dev/null +++ b/quarkus/tests/integration/src/test/java/org/keycloak/it/storage/database/ExternalInfinispanTest.java @@ -0,0 +1,48 @@ +/* + * Copyright 2021 Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.keycloak.it.storage.database; + +import io.quarkus.test.junit.main.Launch; +import org.junit.jupiter.api.Test; +import org.keycloak.common.util.Retry; +import org.keycloak.it.junit5.extension.DistributionTest; +import org.keycloak.it.junit5.extension.InfinispanContainer; +import org.keycloak.it.junit5.extension.WithExternalInfinispan; + +import static io.restassured.RestAssured.when; + +@DistributionTest(keepAlive = true) +@WithExternalInfinispan +public class ExternalInfinispanTest { + + @Test + @Launch({ "start-dev", "--features=multi-site", "--cache=ispn", "--cache-config-file=../../../test-classes/ExternalInfinispan/kcb-infinispan-cache-remote-store-config.xml", "-Djboss.site.name=ISPN" }) + void testLoadBalancerCheckFailure() { + when().get("/lb-check").then() + .statusCode(200); + + InfinispanContainer.remoteCacheManager.administration().removeCache("sessions"); + + // The `lb-check` relies on the Infinispan's persistence check status. By default, Infinispan checks in the background every second that the remote store is available. + // So we'll wait on average about one second here for the check to switch its state. + Retry.execute(() -> { + when().get("/lb-check").then() + .statusCode(503); + }, 10, 200); + } +} diff --git a/quarkus/tests/integration/src/test/resources/ExternalInfinispan/kcb-infinispan-cache-remote-store-config.xml b/quarkus/tests/integration/src/test/resources/ExternalInfinispan/kcb-infinispan-cache-remote-store-config.xml new file mode 100644 index 0000000000..e008424f5b --- /dev/null +++ b/quarkus/tests/integration/src/test/resources/ExternalInfinispan/kcb-infinispan-cache-remote-store-config.xml @@ -0,0 +1,255 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/quarkus/tests/junit5/src/main/java/org/keycloak/it/junit5/extension/CLITestExtension.java b/quarkus/tests/junit5/src/main/java/org/keycloak/it/junit5/extension/CLITestExtension.java index f136610010..97301d807b 100644 --- a/quarkus/tests/junit5/src/main/java/org/keycloak/it/junit5/extension/CLITestExtension.java +++ b/quarkus/tests/junit5/src/main/java/org/keycloak/it/junit5/extension/CLITestExtension.java @@ -35,8 +35,10 @@ import org.keycloak.quarkus.runtime.cli.command.StartDev; import org.keycloak.quarkus.runtime.configuration.KeycloakPropertiesConfigSource; import org.keycloak.quarkus.runtime.configuration.test.TestConfigArgsConfigSource; import org.keycloak.quarkus.runtime.integration.QuarkusPlatform; +import org.testcontainers.containers.GenericContainer; import java.io.IOException; +import java.lang.annotation.Annotation; import java.lang.reflect.Method; import java.util.Arrays; import java.util.Collections; @@ -59,6 +61,7 @@ public class CLITestExtension extends QuarkusMainTestExtension { private KeycloakDistribution dist; private final Set testSysProps = new HashSet<>(); private DatabaseContainer databaseContainer; + private InfinispanContainer infinispanContainer; private CLIResult result; @Override @@ -88,6 +91,7 @@ public class CLITestExtension extends QuarkusMainTestExtension { } configureDatabase(context); + infinispanContainer = configureExternalInfinispan(context); if (distConfig != null) { onKeepServerAlive(context.getRequiredTestMethod().getAnnotation(KeepServerAlive.class)); @@ -192,6 +196,9 @@ public class CLITestExtension extends QuarkusMainTestExtension { databaseContainer.stop(); databaseContainer = null; } + if (infinispanContainer != null && infinispanContainer.isRunning()) { + infinispanContainer.stop(); + } result = null; if (RAW.equals(DistributionType.getCurrent().orElse(RAW))) { if (distConfig != null && !DistributionTest.ReInstall.NEVER.equals(distConfig.reInstall()) && dist != null) { @@ -324,8 +331,24 @@ public class CLITestExtension extends QuarkusMainTestExtension { } } + private static InfinispanContainer configureExternalInfinispan(ExtensionContext context) { + if (getAnnotationFromTestContext(context, WithExternalInfinispan.class) != null) { + InfinispanContainer infinispanContainer = new InfinispanContainer(); + infinispanContainer.start(); + return infinispanContainer; + } + + return null; + } + private static WithDatabase getDatabaseConfig(ExtensionContext context) { - return context.getTestClass().orElse(Object.class).getDeclaredAnnotation(WithDatabase.class); + return getAnnotationFromTestContext(context, WithDatabase.class); + } + + private static T getAnnotationFromTestContext(ExtensionContext context, Class annotationClass) { + return context.getTestClass().map(c -> c.getDeclaredAnnotation(annotationClass)) + .or(() -> context.getTestMethod().map(m -> m.getAnnotation(annotationClass))) + .orElse(null); } private void configureDevServices() { diff --git a/quarkus/tests/junit5/src/main/java/org/keycloak/it/junit5/extension/DatabaseContainer.java b/quarkus/tests/junit5/src/main/java/org/keycloak/it/junit5/extension/DatabaseContainer.java index 08f18030f1..c9ad59e465 100644 --- a/quarkus/tests/junit5/src/main/java/org/keycloak/it/junit5/extension/DatabaseContainer.java +++ b/quarkus/tests/junit5/src/main/java/org/keycloak/it/junit5/extension/DatabaseContainer.java @@ -50,16 +50,9 @@ public class DatabaseContainer { } void configureDistribution(KeycloakDistribution dist) { - if (alias.equals("infinispan")) { - dist.setProperty("storage-hotrod-username", getUsername()); - dist.setProperty("storage-hotrod-password", getPassword()); - dist.setProperty("storage-hotrod-host", container.getHost()); - dist.setProperty("storage-hotrod-port", String.valueOf(container.getMappedPort(11222))); - } else { - dist.setProperty("db-username", getUsername()); - dist.setProperty("db-password", getPassword()); - dist.setProperty("db-url", getJdbcUrl()); - } + dist.setProperty("db-username", getUsername()); + dist.setProperty("db-password", getPassword()); + dist.setProperty("db-url", getJdbcUrl()); } private String getJdbcUrl() { @@ -97,24 +90,10 @@ public class DatabaseContainer { .withInitScript(resolveInitScript()); } - private GenericContainer configureInfinispanUser(GenericContainer infinispanContainer) { - infinispanContainer.addEnv("USER", getUsername()); - infinispanContainer.addEnv("PASS", getPassword()); - return infinispanContainer; - } - private GenericContainer createContainer() { String POSTGRES_IMAGE = System.getProperty("kc.db.postgresql.container.image"); String MARIADB_IMAGE = System.getProperty("kc.db.mariadb.container.image"); String MYSQL_IMAGE = System.getProperty("kc.db.mysql.container.image"); - String INFINISPAN_IMAGE = System.getProperty("kc.infinispan.container.image"); - if (INFINISPAN_IMAGE.matches("quay.io/infinispan/.*-SNAPSHOT")) { - // If the image name ends with SNAPSHOT, someone is trying to use a snapshot release of Infinispan. - // Then switch to the closest match of the Infinispan test container - INFINISPAN_IMAGE = INFINISPAN_IMAGE.replaceAll("quay.io/infinispan/", "quay.io/infinispan-test/"); - INFINISPAN_IMAGE = INFINISPAN_IMAGE.replaceAll("[0-9]*-SNAPSHOT$", "x"); - } - String MSSQL_IMAGE = System.getProperty("kc.db.mssql.container.image"); switch (alias) { @@ -130,14 +109,6 @@ public class DatabaseContainer { case "mssql": DockerImageName MSSQL = DockerImageName.parse(MSSQL_IMAGE).asCompatibleSubstituteFor("sqlserver"); return configureJdbcContainer(new MSSQLServerContainer<>(MSSQL)); - case "infinispan": - GenericContainer infinispanContainer = configureInfinispanUser(new GenericContainer<>(INFINISPAN_IMAGE)) - .withExposedPorts(11222); - // the images in the 'infinispan-test' repository point to tags that are frequently refreshed, therefore, always pull them - if (infinispanContainer.getDockerImageName().startsWith("quay.io/infinispan-test")) { - infinispanContainer.withImagePullPolicy(PullPolicy.alwaysPull()); - } - return infinispanContainer; default: throw new RuntimeException("Unsupported database: " + alias); } diff --git a/quarkus/tests/junit5/src/main/java/org/keycloak/it/junit5/extension/InfinispanContainer.java b/quarkus/tests/junit5/src/main/java/org/keycloak/it/junit5/extension/InfinispanContainer.java new file mode 100644 index 0000000000..c0f73c4d3b --- /dev/null +++ b/quarkus/tests/junit5/src/main/java/org/keycloak/it/junit5/extension/InfinispanContainer.java @@ -0,0 +1,114 @@ +/* + * Copyright 2023 Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.keycloak.it.junit5.extension; + +import org.infinispan.client.hotrod.RemoteCacheManager; +import org.infinispan.client.hotrod.configuration.ClientIntelligence; +import org.infinispan.client.hotrod.configuration.ConfigurationBuilder; +import org.infinispan.commons.configuration.XMLStringConfiguration; +import org.jboss.logging.Logger; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.images.PullPolicy; + +import java.time.Duration; +import java.util.stream.Stream; + +public class InfinispanContainer extends GenericContainer { + + private final Logger LOG = Logger.getLogger(getClass()); + public static final String PORT = System.getProperty("keycloak.externalInfinispan.port", "11222"); + public static final String USERNAME = System.getProperty("keycloak.externalInfinispan.username", "keycloak"); + public static final String PASSWORD = System.getProperty("keycloak.externalInfinispan.password", DatabaseContainer.DEFAULT_PASSWORD); + + public static RemoteCacheManager remoteCacheManager; + + public InfinispanContainer() { + super(getImageName()); + withEnv("USER", USERNAME); + withEnv("PASS", PASSWORD); + withNetworkMode("host"); + + // the images in the 'infinispan-test' repository point to tags that are frequently refreshed, therefore, always pull them + if (getImageName().startsWith("quay.io/infinispan-test")) { + withImagePullPolicy(PullPolicy.alwaysPull()); + } + + + //order of waitingFor and withStartupTimeout matters as the latter sets the timeout for WaitStrategy set by waitingFor + waitingFor(Wait.forLogMessage(".*Infinispan Server.*started in.*", 1)); + withStartupTimeout(Duration.ofMinutes(5)); + } + + private static String getImageName() { + String INFINISPAN_IMAGE = System.getProperty("kc.infinispan.container.image"); + if (INFINISPAN_IMAGE.matches("quay.io/infinispan/.*-SNAPSHOT")) { + // If the image name ends with SNAPSHOT, someone is trying to use a snapshot release of Infinispan. + // Then switch to the closest match of the Infinispan test container + INFINISPAN_IMAGE = INFINISPAN_IMAGE.replaceAll("quay.io/infinispan/", "quay.io/infinispan-test/"); + INFINISPAN_IMAGE = INFINISPAN_IMAGE.replaceAll("[0-9]*-SNAPSHOT$", "x"); + } + + return INFINISPAN_IMAGE; + } + + private void establishHotRodConnection() { + ConfigurationBuilder configBuilder = new ConfigurationBuilder() + .addServers(getContainerIpAddress() + ":11222") + .security() + .authentication() + .username(getUsername()) + .password(getPassword()) + .clientIntelligence(ClientIntelligence.BASIC); + + configBuilder.statistics().enable() + .statistics().jmxEnable(); + + remoteCacheManager = new RemoteCacheManager(configBuilder.build()); + } + + @Override + public void start() { + super.start(); + + establishHotRodConnection(); + + Stream.of("sessions", "actionTokens", "authenticationSessions", "clientSessions", "offlineSessions", "offlineClientSessions", "loginFailures", "work") + .forEach(cacheName -> { + LOG.infof("Creating cache '%s'", cacheName); + createCache(remoteCacheManager, cacheName); + }); + } + + public void createCache(RemoteCacheManager remoteCacheManager, String cacheName) { + String xml = String.format("" , cacheName); + remoteCacheManager.administration().getOrCreateCache(cacheName, new XMLStringConfiguration(xml)); + } + + public String getPort() { + return PORT; + } + + public String getUsername() { + return USERNAME; + } + + public String getPassword() { + return PASSWORD; + } +} diff --git a/quarkus/tests/junit5/src/main/java/org/keycloak/it/junit5/extension/WithExternalInfinispan.java b/quarkus/tests/junit5/src/main/java/org/keycloak/it/junit5/extension/WithExternalInfinispan.java new file mode 100644 index 0000000000..e991c9f6d9 --- /dev/null +++ b/quarkus/tests/junit5/src/main/java/org/keycloak/it/junit5/extension/WithExternalInfinispan.java @@ -0,0 +1,35 @@ +/* + * Copyright 2021 Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.keycloak.it.junit5.extension; + +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * {@link WithExternalInfinispan} is used to start an Infinispan container. + */ +@Target({ElementType.TYPE, ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +@EnabledIfSystemProperty(named = "kc.test.storage.database", matches = "true", disabledReason = "Docker takes too much time and stability depends on the environment. We should try running these tests in CI but isolated.") +public @interface WithExternalInfinispan { + +}