Zero downtime smoke tests

Closes #16481

testsuite/integration-arquillian/HOW-TO-RUN.md

@@ -826,6 +826,19 @@ because this is not UI testing). For debugging purposes you can override the hea
 For changing the hostname in the hostname tests (e.g. [DefaultHostnameTest](https://github.com/keycloak/keycloak/blob/main/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/url/DefaultHostnameTest.java)),
 we rely on [nip.io](https://nip.io) for DNS switching, so tests will work everywhere without fiddling with `etc/hosts` locally. 
 
+### Tips & Tricks:
+Although it _should_ work in general, you may experience an exception like this:
+```
+java.lang.RuntimeException: java.net.UnknownHostException: keycloak.127.0.0.1.nip.io: nodename nor servname provided, 
+or not known at org.keycloak.testsuite.util.OAuthClient.doWellKnownRequest(OAuthClient.java:1032)
+at org.keycloak.testsuite.url.DefaultHostnameTest.assertBackendForcedToFrontendWithMatchingHostname(
+DefaultHostnameTest.java:226)
+...
+```
+when running these tests on your local machine. This happens when something on your machine or network is blocking DNS queries to [nip.io](https://nip.io)
+In order to avoid using external services for DNS resolution, the tests are executed using a local host file by setting the `-Djdk.net.hosts.file=${project.build.testOutputDirectory}/hosts_file` 
+system property.
+
 ## Running base testsuite with Map storage
 
 To run base testsuite with new storage run the following command (this will execute testsuite with ConcurrentHashMap storage):
@@ -933,18 +946,28 @@ export TESTCONTAINERS_RYUK_DISABLED=true #not recommended - see above!
 To use Testcontainers with Docker it is necessary to
 [make Docker available for non-root users](https://docs.docker.com/engine/install/linux-postinstall/).
 
-### Tips & Tricks:
+### Zero downtime tests
-Although it _should_ work in general, you may experience an exception like this:
+
-```
+By default tests are enabled and runs when `map-storage-hotrod`, `map-storage-postgres` or `map-storage-jpa-cockroach` profile is enabled.
-java.lang.RuntimeException: java.net.UnknownHostException: keycloak.127.0.0.1.nip.io: nodename nor servname provided, 
+Other may be added in future. It supports both `auth-server-undertow` and `auth-server-quarkus`.
-or not known at org.keycloak.testsuite.util.OAuthClient.doWellKnownRequest(OAuthClient.java:1032)
+
-at org.keycloak.testsuite.url.DefaultHostnameTest.assertBackendForcedToFrontendWithMatchingHostname(
+#### Default behavior
-DefaultHostnameTest.java:226)
+1. Before test current `auth-server` is stopped as well as ( postgres/crdb/hotrod ) container. 
-...
+2. New ( postgres/crdb/hotrod ) container is spawned using testcontainers.
-```
+3. Legacy keycloak (latest from https://quay.io/) is started.
-when running these tests on your local machine. This happens when something on your machine or network is blocking DNS queries to [nip.io](https://nip.io)
+4. Test realm is imported into legacy keycloak.
-One possible workaround is to add a commonly used public dns server (e.g. 8.8.8.8 for google dns server) to your local 
+5. Current `auth-server` is started.
-networks dns configuration and run the tests. 
+
+#### Notes
+- version of legacy keycloak could be specified by `keycloak.legacy.version.zero.downtime` property
+- legacy keycloak is by default listening on http://localhost:8091 
+- when running only zero downtime tests, e.g. locally, it could be speeded up by skipping start of suite 
+( postgres/crdb/hotrod ) container by `-Dcockroachdb.start-container=false`, `-Dpostgres.start-container=false` 
+or `-Dkeycloak.testsuite.start-hotrod-container=false` and also suite `auth-server` by `-Dkeycloak.testsuite.skip.start.auth.server=true`
+- to run tests with an external instance of ( postgres/crdb/hotrod ) container
+`-Dpostgres.start-container-for-zero-downtime=false`; `-Dcockroachdb.start-container-for-zero-downtime`; `-Dstart-hotrod-container-for-zero-downtime` 
+together with appropriate properties (see sections above).
+
 
 ## FIPS 140-2 testing
 

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/AuthServerTestEnricher.java

@@ -374,6 +374,12 @@ public class AuthServerTestEnricher {
     }
 
     public void startAuthContainer(@Observes(precedence = 0) StartSuiteContainers event) {
+        // this property can be used to skip start of auth-server before suite
+        // it might be useful for running some specific tests locally, e.g. when running standalone ZeroDowtime*Test 
+        if (Boolean.getBoolean("keycloak.testsuite.skip.start.auth.server")) {
+            log.debug("Skipping the start of auth server before suite");
+            return;
+        }
         //frontend-only (either load-balancer or auth-server)
         log.debug("Starting auth server before suite");
 
@@ -734,6 +740,7 @@ public class AuthServerTestEnricher {
         }
 
         TestContext testContext = testContextProducer.get();
+        testContext.runAfterClassActions();
 
         Keycloak adminClient = testContext.getAdminClient();
         KeycloakTestingClient testingClient = testContext.getTestingClient();

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/CockroachdbContainerProvider.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright 2023 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.arquillian;
+
+import java.lang.annotation.Annotation;
+import org.jboss.arquillian.container.spi.event.StartSuiteContainers;
+import org.jboss.arquillian.core.api.annotation.Observes;
+import org.jboss.arquillian.test.api.ArquillianResource;
+import org.jboss.arquillian.test.spi.enricher.resource.ResourceProvider;
+import org.jboss.arquillian.test.spi.event.suite.AfterSuite;
+import org.jboss.logging.Logger;
+import org.testcontainers.containers.CockroachContainer;
+import org.testcontainers.utility.DockerImageName;
+
+public class CockroachdbContainerProvider implements ResourceProvider {
+
+    private static final Logger log = Logger.getLogger(CockroachdbContainerProvider.class);
+
+    private static final Boolean START_CONTAINER = Boolean.valueOf(System.getProperty("cockroachdb.start-container", "false"));
+    private static final String COCKROACHDB_DOCKER_IMAGE_NAME = System.getProperty("keycloak.map.storage.cockroachdb.docker.image", "cockroachdb/cockroach:v22.1.0");
+
+    public static final String COCKROACHDB_DB_USER = System.getProperty("keycloak.map.storage.connectionsJpa.user", "keycloak");
+    public static final String COCKROACHDB_DB_PASSWORD = System.getProperty("keycloak.map.storage.connectionsJpa.password", "pass");
+
+    private static CockroachContainer cockroachDbContainer;
+
+    public void beforeContainerStarted(@Observes(precedence = 1) StartSuiteContainers event) {
+        if (START_CONTAINER) {
+            cockroachDbContainer = createContainer();
+            cockroachDbContainer.start();
+
+            System.setProperty("keycloak.map.storage.connectionsJpa.url", cockroachDbContainer.getJdbcUrl());
+
+            log.infof("DatabaseInfo: %s, user=%s, pass=%s", cockroachDbContainer.getJdbcUrl(), COCKROACHDB_DB_USER, COCKROACHDB_DB_PASSWORD);
+        }
+    }
+
+    public void afterSuite(@Observes(precedence = 4) AfterSuite event) {
+        if (START_CONTAINER) {
+            cockroachDbContainer.stop();
+        }
+    }
+
+    public CockroachContainer createContainer() {
+        return new CockroachContainer(DockerImageName.parse(COCKROACHDB_DOCKER_IMAGE_NAME).asCompatibleSubstituteFor("cockroachdb"))
+                // Using the environment variables for now where using the withXXX() method is not supported, yet.
+                // https://github.com/testcontainers/testcontainers-java/issues/6299
+                .withEnv("COCKROACH_DATABASE", "keycloak")
+                .withEnv("COCKROACH_USER", COCKROACHDB_DB_USER)
+                // password is not used/supported in insecure mode
+                .withCommand("start-single-node", "--insecure");
+    }
+
+    public CockroachContainer getContainer() {
+        return cockroachDbContainer;
+    }
+
+    @Override
+    public boolean canProvide(Class<?> type) {
+        return type.equals(CockroachdbContainerProvider.class);
+    }
+
+    @Override
+    public Object lookup(ArquillianResource ar, Annotation... antns) {
+        return this;
+    }
+}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/CockroachdbContainerTestEnricher.java

@@ -1,52 +0,0 @@
-/*
- * Copyright 2022 Red Hat, Inc. and/or its affiliates
- * and other contributors as indicated by the @author tags.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.keycloak.testsuite.arquillian;
-
-import org.jboss.arquillian.container.spi.event.StartSuiteContainers;
-import org.jboss.arquillian.core.api.annotation.Observes;
-import org.jboss.arquillian.test.spi.event.suite.AfterSuite;
-import org.testcontainers.containers.CockroachContainer;
-import org.testcontainers.utility.DockerImageName;
-
-public class CockroachdbContainerTestEnricher {
-
-    private static final Boolean START_CONTAINER = Boolean.valueOf(System.getProperty("cockroachdb.start-container", "false"));
-    private static final String COCKROACHDB_DOCKER_IMAGE_NAME = System.getProperty("keycloak.map.storage.cockroachdb.docker.image", "cockroachdb/cockroach:v22.1.0");
-    private static final CockroachContainer COCKROACHDB_CONTAINER = new CockroachContainer(DockerImageName.parse(COCKROACHDB_DOCKER_IMAGE_NAME).asCompatibleSubstituteFor("cockroachdb"));
-    private static final String COCKROACHDB_DB_USER = System.getProperty("keycloak.map.storage.connectionsJpa.user", "keycloak");
-
-    public void beforeContainerStarted(@Observes(precedence = 1) StartSuiteContainers event) {
-        if (START_CONTAINER) {
-            COCKROACHDB_CONTAINER
-                    // Using the environment variables for now where using the withXXX() method is not supported, yet.
-                    // https://github.com/testcontainers/testcontainers-java/issues/6299
-                    .withEnv("COCKROACH_DATABASE", "keycloak")
-                    .withEnv("COCKROACH_USER", COCKROACHDB_DB_USER)
-                    // password is not used/supported in insecure mode
-                    .withCommand("start-single-node", "--insecure")
-                    .start();
-
-            System.setProperty("keycloak.map.storage.connectionsJpa.url", COCKROACHDB_CONTAINER.getJdbcUrl());
-        }
-    }
-
-    public void afterSuite(@Observes(precedence = 4) AfterSuite event) {
-        if (START_CONTAINER) {
-            COCKROACHDB_CONTAINER.stop();
-        }
-    }
-}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/HotRodContainerProvider.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright 2023 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.arquillian;
+
+import java.lang.annotation.Annotation;
+import org.jboss.arquillian.container.spi.event.StartSuiteContainers;
+import org.jboss.arquillian.core.api.annotation.Observes;
+import org.jboss.arquillian.test.api.ArquillianResource;
+import org.jboss.arquillian.test.spi.enricher.resource.ResourceProvider;
+import org.jboss.arquillian.test.spi.event.suite.AfterSuite;
+import org.keycloak.testsuite.util.InfinispanContainer;
+
+
+public class HotRodContainerProvider implements ResourceProvider {
+
+    public static final String HOT_ROD_STORE_HOST_PROPERTY = "keycloak.connectionsHotRod.host";
+
+    public static final boolean HOT_ROD_START_CONTAINER = Boolean.parseBoolean(System.getProperty("keycloak.testsuite.start-hotrod-container", "false"));
+
+    private static InfinispanContainer infinispanContainer;
+
+    public void beforeContainerStarted(@Observes(precedence = 1) StartSuiteContainers event) {
+        if (!HOT_ROD_START_CONTAINER) return;
+        infinispanContainer = new InfinispanContainer();
+        infinispanContainer.start();
+
+        // Add env variable, so it can be picked up by Keycloak
+        System.setProperty(HOT_ROD_STORE_HOST_PROPERTY, infinispanContainer.getHost());
+    }
+
+    public void afterSuite(@Observes(precedence = 4) AfterSuite event) {
+        if (!HOT_ROD_START_CONTAINER) return;
+        if (infinispanContainer != null) infinispanContainer.stop();
+    }
+
+    public InfinispanContainer getContainer() {
+        return infinispanContainer;
+    }
+
+    @Override
+    public boolean canProvide(Class<?> type) {
+        return type.equals(HotRodContainerProvider.class);
+    }
+
+    @Override
+    public Object lookup(ArquillianResource ar, Annotation... antns) {
+        return this;
+    }
+}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/HotRodStoreTestEnricher.java

@@ -1,30 +0,0 @@
-package org.keycloak.testsuite.arquillian;
-
-import org.jboss.arquillian.container.spi.event.StartSuiteContainers;
-import org.jboss.arquillian.core.api.annotation.Observes;
-import org.jboss.arquillian.test.spi.event.suite.AfterSuite;
-import org.keycloak.testsuite.util.InfinispanContainer;
-
-
-public class HotRodStoreTestEnricher {
-
-    public static final String HOT_ROD_STORE_HOST_PROPERTY = "keycloak.connectionsHotRod.host";
-
-    public static final boolean HOT_ROD_START_CONTAINER = Boolean.parseBoolean(System.getProperty("keycloak.testsuite.start-hotrod-container", "false"));
-
-    private InfinispanContainer hotRodContainer;
-
-    public void beforeContainerStarted(@Observes(precedence = 1) StartSuiteContainers event) {
-        if (!HOT_ROD_START_CONTAINER) return;
-        hotRodContainer = new InfinispanContainer();
-        hotRodContainer.start();
-
-        // Add env variable, so it can be picked up by Keycloak
-        System.setProperty(HOT_ROD_STORE_HOST_PROPERTY, hotRodContainer.getHost());
-    }
-
-    public void afterSuite(@Observes(precedence = 4) AfterSuite event) {
-        if (!HOT_ROD_START_CONTAINER) return;
-        if (hotRodContainer != null) hotRodContainer.stop();
-    }
-}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/KeycloakArquillianExtension.java

@@ -56,7 +56,11 @@ public class KeycloakArquillianExtension implements LoadableExtension {
                 .service(ResourceProvider.class, TestContextProvider.class)
                 .service(ResourceProvider.class, AdminClientProvider.class)
                 .service(ResourceProvider.class, OAuthClientProvider.class)
-                .service(ResourceProvider.class, LoadBalancerControllerProvider.class);
+                .service(ResourceProvider.class, LoadBalancerControllerProvider.class)
+                .service(ResourceProvider.class, PostgresContainerProvider.class)
+                .service(ResourceProvider.class, CockroachdbContainerProvider.class)
+                .service(ResourceProvider.class, HotRodContainerProvider.class)
+                .service(ResourceProvider.class, LegacyKeycloakContainerProvider.class);
 
         builder
                 .service(DeploymentScenarioGenerator.class, DeploymentTargetModifier.class)
@@ -66,9 +70,9 @@ public class KeycloakArquillianExtension implements LoadableExtension {
                 .observer(AuthServerTestEnricher.class)
                 .observer(AppServerTestEnricher.class)
                 .observer(CrossDCTestEnricher.class)
-                .observer(HotRodStoreTestEnricher.class)
+                .observer(HotRodContainerProvider.class)
-                .observer(PostgresContainerTestEnricher.class)
+                .observer(PostgresContainerProvider.class)
-                .observer(CockroachdbContainerTestEnricher.class)
+                .observer(CockroachdbContainerProvider.class)
                 .observer(H2TestEnricher.class);
         builder
                 .service(TestExecutionDecider.class, MigrationTestExecutionDecider.class)

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/LegacyKeycloakContainerProvider.java

@@ -0,0 +1,111 @@
+/*
+ * Copyright 2023 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.arquillian;
+
+import java.lang.annotation.Annotation;
+import org.jboss.arquillian.test.api.ArquillianResource;
+import org.jboss.arquillian.test.spi.enricher.resource.ResourceProvider;
+import org.keycloak.testsuite.util.InfinispanContainer;
+import org.keycloak.testsuite.util.LegacyKeycloakContainer;
+
+public class LegacyKeycloakContainerProvider implements ResourceProvider {
+
+    public static final ZeroDowntimeContainer CONTAINER = processSystemProperties();
+    public enum ZeroDowntimeContainer {
+        POSTGRES,
+        COCKROACH,
+        HOTROD,
+        NONE;
+    }
+
+    private final String legacyKeycloakPort = System.getProperty("keycloak.legacy.port", "8091");
+
+    @Override
+    public boolean canProvide(Class<?> type) {
+        return type.equals(LegacyKeycloakContainerProvider.class);
+    }
+
+    @Override
+    public Object lookup(ArquillianResource ar, Annotation... antns) {
+        return this;
+    }
+
+    private static ZeroDowntimeContainer processSystemProperties() {
+        String mapStorageProfileconfig = System.getProperty("auth.server.quarkus.mapStorage.profile.config");
+        if (mapStorageProfileconfig == null) return ZeroDowntimeContainer.NONE;
+
+        switch (mapStorageProfileconfig) {
+
+            case "jpa":
+                String storageConnectionsVendor = System.getProperty("keycloak.storage.connections.vendor");
+                if (storageConnectionsVendor == null) return ZeroDowntimeContainer.NONE;
+
+                switch (storageConnectionsVendor) {
+
+                    case "postgres":
+                        return ZeroDowntimeContainer.POSTGRES;
+                    case "cockroach":
+                        return ZeroDowntimeContainer.COCKROACH;
+                    default:
+                        return ZeroDowntimeContainer.NONE;
+                }
+
+            case "hotrod":
+                return ZeroDowntimeContainer.HOTROD;
+
+            default:
+                return ZeroDowntimeContainer.NONE;
+        }
+    }
+
+    private LegacyKeycloakContainer createLegacyKeycloakContainer() {
+        return new LegacyKeycloakContainer(System.getProperty("keycloak.legacy.version.zero.downtime", "latest"));
+    }
+
+    public LegacyKeycloakContainer get() {
+        switch (CONTAINER) {
+
+            case POSTGRES:
+                return createLegacyKeycloakContainer().withCommand("start-dev", 
+                        "--storage=jpa",
+                        "--http-port=" + legacyKeycloakPort,
+                        "--db-url=" + System.getProperty("keycloak.map.storage.connectionsJpa.url"),
+                        "--db-username=" + PostgresContainerProvider.POSTGRES_DB_USER,
+                        "--db-password=" + PostgresContainerProvider.POSTGRES_DB_PASSWORD);
+
+            case COCKROACH:
+                return createLegacyKeycloakContainer().withCommand("start-dev", 
+                        "--storage=jpa",
+                        "--http-port=" + legacyKeycloakPort,
+                        "--db-url=" + System.getProperty("keycloak.map.storage.connectionsJpa.url"),
+                        "--db-username=" + CockroachdbContainerProvider.COCKROACHDB_DB_USER,
+                        "--db-password=" + CockroachdbContainerProvider.COCKROACHDB_DB_PASSWORD);
+
+            case HOTROD:
+                return createLegacyKeycloakContainer().withCommand("start-dev", 
+                        "--storage=hotrod",
+                        "--http-port=" + legacyKeycloakPort,
+                        "--storage-hotrod-host=" + System.getProperty(HotRodContainerProvider.HOT_ROD_STORE_HOST_PROPERTY),
+                        "--storage-hotrod-port=" + InfinispanContainer.PORT,
+                        "--storage-hotrod-username=" + InfinispanContainer.USERNAME,
+                        "--storage-hotrod-password=" + InfinispanContainer.PASSWORD);
+            case NONE:
+            default:
+                return null;
+        }
+    }
+}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/PostgresContainerProvider.java

@@ -0,0 +1,80 @@
+/*
+ * Copyright 2023 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.arquillian;
+
+import java.lang.annotation.Annotation;
+import org.jboss.arquillian.container.spi.event.StartSuiteContainers;
+import org.jboss.arquillian.core.api.annotation.Observes;
+import org.jboss.arquillian.test.api.ArquillianResource;
+import org.jboss.arquillian.test.spi.enricher.resource.ResourceProvider;
+import org.jboss.arquillian.test.spi.event.suite.AfterSuite;
+import org.jboss.logging.Logger;
+import org.testcontainers.containers.PostgreSQLContainer;
+import org.testcontainers.utility.DockerImageName;
+
+public class PostgresContainerProvider implements ResourceProvider {
+
+    private final Logger log = Logger.getLogger(PostgresContainerProvider.class);
+
+    private static final Boolean START_CONTAINER = Boolean.valueOf(System.getProperty("postgres.start-container", "false"));
+    private static final String POSTGRES_DOCKER_IMAGE_NAME = System.getProperty("keycloak.map.storage.postgres.docker.image", "postgres:alpine");
+
+    public static final String POSTGRES_DB_USER = System.getProperty("keycloak.map.storage.connectionsJpa.user", "keycloak");
+    public static final String POSTGRES_DB_PASSWORD = System.getProperty("keycloak.map.storage.connectionsJpa.password", "pass");
+
+    private static PostgreSQLContainer postgresContainer;
+
+    public void beforeContainerStarted(@Observes(precedence = 1) StartSuiteContainers event) {
+        if (START_CONTAINER) {
+            postgresContainer = createContainer();
+            postgresContainer.start();
+            
+            System.setProperty("keycloak.map.storage.connectionsJpa.url", postgresContainer.getJdbcUrl());
+
+            log.infof("DatabaseInfo: %s, user=%s, pass=%s", postgresContainer.getJdbcUrl(), POSTGRES_DB_USER, POSTGRES_DB_PASSWORD);
+        }
+    }
+
+    public void afterSuite(@Observes(precedence = 4) AfterSuite event) {
+        if (START_CONTAINER) {
+            postgresContainer.stop();
+        }
+    }
+
+    public PostgreSQLContainer createContainer() {
+        return new PostgreSQLContainer(DockerImageName.parse(POSTGRES_DOCKER_IMAGE_NAME).asCompatibleSubstituteFor("postgres"))
+                .withDatabaseName("keycloak")
+                .withUsername(POSTGRES_DB_USER)
+                .withPassword(POSTGRES_DB_PASSWORD);
+    }
+
+    public PostgreSQLContainer getContainer() {
+        return postgresContainer;
+    }
+
+    @Override
+    public boolean canProvide(Class<?> type) {
+        return type.equals(PostgresContainerProvider.class);
+    }
+
+    @Override
+    public Object lookup(ArquillianResource ar, Annotation... antns) {
+        return this;
+    }
+
+    
+}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/PostgresContainerTestEnricher.java

@@ -1,50 +0,0 @@
-/*
- * Copyright 2022 Red Hat, Inc. and/or its affiliates
- * and other contributors as indicated by the @author tags.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.keycloak.testsuite.arquillian;
-
-import org.jboss.arquillian.container.spi.event.StartSuiteContainers;
-import org.jboss.arquillian.core.api.annotation.Observes;
-import org.jboss.arquillian.test.spi.event.suite.AfterSuite;
-import org.testcontainers.containers.PostgreSQLContainer;
-import org.testcontainers.utility.DockerImageName;
-
-public class PostgresContainerTestEnricher {
-
-    private static final Boolean START_CONTAINER = Boolean.valueOf(System.getProperty("postgres.start-container", "false"));
-    private static final String POSTGRES_DOCKER_IMAGE_NAME = System.getProperty("keycloak.map.storage.postgres.docker.image", "postgres:alpine");
-    private static final PostgreSQLContainer POSTGRES_CONTAINER = new PostgreSQLContainer(DockerImageName.parse(POSTGRES_DOCKER_IMAGE_NAME).asCompatibleSubstituteFor("postgres"));
-    private static final String POSTGRES_DB_USER = System.getProperty("keycloak.map.storage.connectionsJpa.user", "keycloak");
-    private static final String POSTGRES_DB_PASSWORD = System.getProperty("keycloak.map.storage.connectionsJpa.password", "pass");
-
-    public void beforeContainerStarted(@Observes(precedence = 1) StartSuiteContainers event) {
-        if (START_CONTAINER) {
-            POSTGRES_CONTAINER
-                    .withDatabaseName("keycloak")
-                    .withUsername(POSTGRES_DB_USER)
-                    .withPassword(POSTGRES_DB_PASSWORD)
-                    .start();
-
-            System.setProperty("keycloak.map.storage.connectionsJpa.url", POSTGRES_CONTAINER.getJdbcUrl());
-        }
-    }
-
-    public void afterSuite(@Observes(precedence = 4) AfterSuite event) {
-        if (START_CONTAINER) {
-            POSTGRES_CONTAINER.stop();
-        }
-    }
-}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/TestContext.java

@@ -18,6 +18,7 @@ package org.keycloak.testsuite.arquillian;
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -58,6 +59,8 @@ public final class TestContext {
     // Key is realmName, value are objects to clean after the test method
     private final Map<String, TestCleanup> cleanups = new ConcurrentHashMap<>();
 
+    private final Set<Runnable> afterClassActions = new HashSet<>();
+
     public TestContext(SuiteContext suiteContext, Class testClass) {
         this.suiteContext = suiteContext;
         this.testClass = testClass;
@@ -201,6 +204,14 @@ public final class TestContext {
         return cleanups;
     }
 
+    public void registerAfterClassAction(Runnable afterClassAction) {
+        afterClassActions.add(afterClassAction);
+    }
+
+    public void runAfterClassActions() {
+        afterClassActions.forEach(Runnable::run);
+        afterClassActions.clear();
+    }
 
     public String getAppServerContainerName() {
         if (isAdapterContainerEnabled()) { //standalone app server

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/InfinispanContainer.java

@@ -18,7 +18,7 @@
 package org.keycloak.testsuite.util;
 
 import org.jboss.logging.Logger;
-import org.keycloak.testsuite.arquillian.HotRodStoreTestEnricher;
+import org.keycloak.testsuite.arquillian.HotRodContainerProvider;
 import org.testcontainers.containers.GenericContainer;
 import org.testcontainers.containers.wait.strategy.Wait;
 import org.testcontainers.utility.MountableFile;
@@ -33,10 +33,10 @@ import java.util.regex.Pattern;
 public class InfinispanContainer extends GenericContainer<InfinispanContainer> {
 
     private final Logger LOG = Logger.getLogger(getClass());
-    private static final String PORT = System.getProperty("keycloak.connectionsHotRod.port", "11222");
+    public static final String PORT = System.getProperty("keycloak.connectionsHotRod.port", "11222");
-    private static String HOST = System.getProperty(HotRodStoreTestEnricher.HOT_ROD_STORE_HOST_PROPERTY);
+    private static String HOST = System.getProperty(HotRodContainerProvider.HOT_ROD_STORE_HOST_PROPERTY);
-    private static final String USERNAME = System.getProperty("keycloak.connectionsHotRod.username", "admin");
+    public static final String USERNAME = System.getProperty("keycloak.connectionsHotRod.username", "admin");
-    private static final String PASSWORD = System.getProperty("keycloak.connectionsHotRod.password", "admin");
+    public static final String PASSWORD = System.getProperty("keycloak.connectionsHotRod.password", "admin");
 
     private static final String ZERO_TO_255
             = "(\\d{1,2}|(0|1)\\"
@@ -67,10 +67,12 @@ public class InfinispanContainer extends GenericContainer<InfinispanContainer> {
         MountableFile mountableFile = MountableFile.forHostPath(timeTaskPath, 0666);
         withCopyFileToContainer(mountableFile, "/opt/infinispan/server/lib/integration-arquillian-testsuite-providers.jar");
 
-        withStartupTimeout(Duration.ofMinutes(5));
+        //order of waitingFor and withStartupTimeout matters as the latter sets the timeout for WaitStrategy set by waitingFor
         waitingFor(Wait.forLogMessage(".*Infinispan Server.*started in.*", 1));
+        withStartupTimeout(Duration.ofMinutes(5));
     }
 
+    @Override
     public String getHost() {
         if (HOST == null && this.isRunning()) {
             Matcher matcher = IP_ADDRESS_PATTERN.matcher(getLogs());
@@ -88,11 +90,6 @@ public class InfinispanContainer extends GenericContainer<InfinispanContainer> {
         return HOST;
     }
 
-    @Override
-    public void start() {
-        super.start();
-    }
-
     public String getPort() {
         return PORT;
     }

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/LegacyKeycloakContainer.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2023 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.testsuite.util;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.time.Duration;
+import org.keycloak.util.JsonSerialization;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.wait.strategy.Wait;
+
+public class LegacyKeycloakContainer extends GenericContainer<LegacyKeycloakContainer> {
+
+    private static final String USERNAME = "admin";
+    private static final String PASSWORD = "admin";
+
+    public LegacyKeycloakContainer(String tagName) {
+        super("quay.io/keycloak/keycloak" + getManifestDigestOfImageByName(tagName));
+
+        withEnv("KEYCLOAK_ADMIN", USERNAME);
+        withEnv("KEYCLOAK_ADMIN_PASSWORD", PASSWORD);
+        withNetworkMode("host");
+
+        //order of waitingFor and withStartupTimeout matters as the latter sets the timeout for WaitStrategy set by waitingFor
+        waitingFor(Wait.forLogMessage(".*Running the server in development mode..*", 1));
+        withStartupTimeout(Duration.ofMinutes(5));
+    }
+
+    /**
+     * For tag names as "latest" or "nightly" it may happen that some older version keycloak could be cached.
+     * Therefore fetching container by its name wouldn't be reliable.
+     * 
+     * This method obtains image "manifest_digest" by the name from quay.io and returns it with prefix "@".
+     * It can be then used to fetch the container with: 
+     * 
+     * <p>quay.io/keycloak/keycloak@sha256:...</p>
+     * 
+     * @param tagName Name of the image tag.
+     * @return "manifest_digest" of the image if "latest" or "nightly" with prefix "@", {@code tagName} with ":" prefix otherwise
+     */
+    private static String getManifestDigestOfImageByName(String tagName) {
+        if ("latest".equals(tagName) || "nightly".equals(tagName)) {
+
+            try {
+                URI uri = new URI("https://quay.io/api/v1/repository/keycloak/keycloak/tag/?specificTag=" + tagName);
+                HttpResponse<String> response = HttpClient.newBuilder()
+                        .followRedirects(HttpClient.Redirect.ALWAYS)
+                        .build()
+                        .send(HttpRequest.newBuilder(uri).GET().build(), HttpResponse.BodyHandlers.ofString());
+                if (response.statusCode() == HttpURLConnection.HTTP_OK) {
+                    JsonNode manifestDigest = JsonSerialization.mapper.readTree(response.body()).findValue("manifest_digest");
+                    if (manifestDigest != null) {
+                        return "@" + manifestDigest.asText();
+                    }
+                }
+                throw new RuntimeException(String.format("Unable to get manifest_digest for image with tag %s from quay.io. Response: %d, %s. ", tagName, response.statusCode(), response.body()));
+            } catch (URISyntaxException | IOException | InterruptedException ex) {
+                throw new RuntimeException(ex);
+            }
+
+        } else {
+            return ":" + tagName;
+        }
+    }
+}

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/util/ClientScopeBuilder.java

@@ -23,13 +23,17 @@ import org.keycloak.representations.idm.ClientScopeRepresentation;
  */
 public class ClientScopeBuilder {
 
-    private ClientScopeRepresentation rep;
+    private final ClientScopeRepresentation rep;
 
     public static ClientScopeBuilder create() {
         ClientScopeRepresentation rep = new ClientScopeRepresentation();
         return new ClientScopeBuilder(rep);
     }
 
+    public static ClientScopeBuilder edit(ClientScopeRepresentation rep) {
+        return new ClientScopeBuilder(rep);
+    }
+
     private ClientScopeBuilder(ClientScopeRepresentation rep) {
         this.rep = rep;
     }
@@ -43,6 +47,11 @@ public class ClientScopeBuilder {
         return this;
     }
 
+    public ClientScopeBuilder description(String description) {
+        rep.setDescription(description);
+        return this;
+    }
+
     public ClientScopeBuilder protocol(String protocol) {
         rep.setProtocol(protocol);
         return this;

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/util/GroupBuilder.java

@@ -34,6 +34,10 @@ public class GroupBuilder {
         return new GroupBuilder(rep);
     }
 
+    public static GroupBuilder edit(GroupRepresentation rep) {
+        return new GroupBuilder(rep);
+    }
+
     private GroupBuilder(GroupRepresentation rep) {
         this.rep = rep;
     }

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/util/RealmBuilder.java

@@ -59,6 +59,11 @@ public class RealmBuilder {
         return this;
     }
 
+    public RealmBuilder displayName(String displayName) {
+        rep.setDisplayName(displayName);
+        return this;
+    }
+
     public RealmBuilder publicKey(String publicKey) {
         rep.setPublicKey(publicKey);
         return this;

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/util/RoleBuilder.java

@@ -31,13 +31,18 @@ import java.util.Map;
  */
 public class RoleBuilder {
 
-    private RoleRepresentation rep = new RoleRepresentation();
+    private final RoleRepresentation rep;
 
     public static RoleBuilder create() {
-        return new RoleBuilder();
+        return new RoleBuilder(new RoleRepresentation());
     }
 
-    private RoleBuilder() {
+    public static RoleBuilder edit(RoleRepresentation rep) {
+        return new RoleBuilder(rep);
+    }
+
+    private RoleBuilder(RoleRepresentation rep) {
+        this.rep = rep;
     }
 
     public RoleBuilder id(String id) {
@@ -84,7 +89,7 @@ public class RoleBuilder {
         checkCompositesNull();
 
         if (rep.getComposites().getRealm() == null) {
-            rep.getComposites().setRealm(new HashSet<String>());
+            rep.getComposites().setRealm(new HashSet<>());
         }
 
         rep.getComposites().getRealm().add(compositeRole);
@@ -99,11 +104,11 @@ public class RoleBuilder {
         checkCompositesNull();
 
         if (rep.getComposites().getClient() == null) {
-            rep.getComposites().setClient(new HashMap<String, List<String>>());
+            rep.getComposites().setClient(new HashMap<>());
         }
 
         if (rep.getComposites().getClient().get(client) == null) {
-            rep.getComposites().getClient().put(client, new LinkedList<String>());
+            rep.getComposites().getClient().put(client, new LinkedList<>());
         }
 
         rep.getComposites().getClient().get(client).add(compositeRole);

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/zerodowntime/ZeroDowntimeTest.java

@@ -0,0 +1,333 @@
+/*
+ * Copyright 2023 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.zerodowntime;
+
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.time.Duration;
+import java.util.List;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import org.jboss.arquillian.container.test.api.ContainerController;
+import org.jboss.arquillian.test.api.ArquillianResource;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Test;
+import org.keycloak.admin.client.Keycloak;
+import org.keycloak.admin.client.KeycloakBuilder;
+import org.keycloak.common.util.Retry;
+import org.keycloak.models.Constants;
+import org.keycloak.representations.idm.ClientRepresentation;
+import org.keycloak.representations.idm.ClientScopeRepresentation;
+import org.keycloak.representations.idm.GroupRepresentation;
+import org.keycloak.representations.idm.RealmRepresentation;
+import org.keycloak.representations.idm.RoleRepresentation;
+import org.keycloak.representations.idm.UserRepresentation;
+import org.keycloak.testsuite.AbstractKeycloakTest;
+import org.keycloak.testsuite.arquillian.CockroachdbContainerProvider;
+import org.keycloak.testsuite.arquillian.HotRodContainerProvider;
+import org.keycloak.testsuite.arquillian.LegacyKeycloakContainerProvider;
+import org.keycloak.testsuite.arquillian.LegacyKeycloakContainerProvider.ZeroDowntimeContainer;
+import static org.keycloak.testsuite.arquillian.LegacyKeycloakContainerProvider.ZeroDowntimeContainer.COCKROACH;
+import static org.keycloak.testsuite.arquillian.LegacyKeycloakContainerProvider.ZeroDowntimeContainer.HOTROD;
+import static org.keycloak.testsuite.arquillian.LegacyKeycloakContainerProvider.ZeroDowntimeContainer.NONE;
+import static org.keycloak.testsuite.arquillian.LegacyKeycloakContainerProvider.ZeroDowntimeContainer.POSTGRES;
+import org.keycloak.testsuite.arquillian.PostgresContainerProvider;
+import org.keycloak.testsuite.auth.page.AuthRealm;
+import org.keycloak.testsuite.updaters.SetSystemProperty;
+import org.keycloak.testsuite.util.AdminClientUtil;
+import org.keycloak.testsuite.util.ClientBuilder;
+import org.keycloak.testsuite.util.ClientScopeBuilder;
+import org.keycloak.testsuite.util.GroupBuilder;
+import org.keycloak.testsuite.util.InfinispanContainer;
+import org.keycloak.testsuite.util.LegacyKeycloakContainer;
+import org.keycloak.testsuite.util.RealmBuilder;
+import org.keycloak.testsuite.util.RoleBuilder;
+import org.keycloak.testsuite.util.RolesBuilder;
+import org.keycloak.testsuite.util.UserBuilder;
+import org.testcontainers.containers.CockroachContainer;
+import org.testcontainers.containers.PostgreSQLContainer;
+
+/**
+ * It tests that object stored by legacy version of keycloak could be read by
+ * current one, then object is modified, therefore stored in current version and
+ * then it's verified it could be read by legacy keycloak.
+ *
+ * See HOW-TO-RUN.md for some additional notes.
+ *
+ * @author vramik
+ */
+public class ZeroDowntimeTest extends AbstractKeycloakTest {
+
+    @ArquillianResource private ContainerController controller;
+    @ArquillianResource private LegacyKeycloakContainerProvider legacyKeycloakServerProvider;
+    @ArquillianResource private PostgresContainerProvider postgresProvider;
+    @ArquillianResource private CockroachdbContainerProvider cockroachdbProvider;
+    @ArquillianResource private HotRodContainerProvider hotrodProvider;
+
+    private final String legacyKeycloakHost = System.getProperty("keycloak.legacy.host", "localhost");
+    private final String legacyKeycloakPort = System.getProperty("keycloak.legacy.port", "8091");
+    private final String legacyKeycloakUrl = "http://" + legacyKeycloakHost + ":" + legacyKeycloakPort;
+
+    private String currentAuthServer;
+    private SetSystemProperty sysProp;
+    private static boolean initialized;
+    private static Keycloak legacyAdminClient;
+
+    // junit4 enforces to use static method when using @BeforeClass or @ClassRule
+    // TODO: after upgrade to junit5: replace @Before with @BeforeAll and get rid of the "initialized" flag
+    @Before
+    public void before() throws Exception {
+        Assume.assumeFalse(LegacyKeycloakContainerProvider.CONTAINER.equals(ZeroDowntimeContainer.NONE));
+
+        if (initialized) return;
+
+        currentAuthServer = suiteContext.getAuthServerInfo().getArquillianContainer().getName();
+
+        // stop current auth-server
+        if (controller.isStarted(currentAuthServer)) {
+            log.debug("Stopping auth server");
+            controller.stop(currentAuthServer);
+        }
+
+        // stop and create new container
+        handleContainer();
+
+        //start legacy container
+        LegacyKeycloakContainer legacyKeycloakServer = legacyKeycloakServerProvider.get();
+        legacyKeycloakServer.start();
+        legacyAdminClient = KeycloakBuilder.builder()
+                .serverUrl(legacyKeycloakUrl)
+                .realm(AuthRealm.MASTER)
+                .username(AuthRealm.ADMIN)
+                .password(AuthRealm.ADMIN)
+                .clientId(Constants.ADMIN_CLI_CLIENT_ID)
+                .resteasyClient(AdminClientUtil.createResteasyClient())
+                .build();
+
+        // import test realm
+        legacyAdminClient.realms().create(RealmBuilder.create()
+                .name(AuthRealm.TEST)
+                .attribute("attr", "val")
+                .client(ClientBuilder.create()
+                    .clientId("client"))
+                .clientScope(ClientScopeBuilder.create().name("client_scope"))
+                .roles(RolesBuilder.create()
+                        .clientRole("client", RoleBuilder.create().name("client_role").singleAttribute("attr", "val").build())
+                        .realmRole(RoleBuilder.create().name("realm_role").singleAttribute("attr", "val").build()))
+                .group(GroupBuilder.create().name("group").build())
+                .user(UserBuilder.create().username("user").addAttribute("attr", "val"))
+                .build());
+
+        // start current auth-server
+        controller.start(currentAuthServer);
+        reconnectAdminClient();
+
+        testContext.registerAfterClassAction(() -> {
+            legacyKeycloakServer.stop();
+            if (sysProp != null) sysProp.revert();
+        });
+
+        initialized = true;
+    }
+
+    private void handleContainer() {
+        switch (LegacyKeycloakContainerProvider.CONTAINER) {
+            case POSTGRES:
+                handlePostgres();
+                break;
+            case COCKROACH:
+                handleCockroach();
+                break;
+            case HOTROD:
+                handleHotRod();
+                break;
+            case NONE:
+            default:
+                throw new IllegalStateException("Unknown container!");
+        }
+    }
+
+    private void handlePostgres() {
+        PostgreSQLContainer postgres = postgresProvider.getContainer();
+        //if suite container is running stop it
+        if (postgres != null && postgres.isRunning()) {
+            postgres.stop();
+        }
+        if (Boolean.parseBoolean(System.getProperty("postgres.start-container-for-zero-downtime", "true"))) {
+            postgres = postgresProvider.createContainer();
+            postgres.start();
+
+            log.infof("DatabaseInfo: %s, user=%s, pass=%s", postgres.getJdbcUrl(), PostgresContainerProvider.POSTGRES_DB_USER, PostgresContainerProvider.POSTGRES_DB_PASSWORD);
+
+            sysProp = new SetSystemProperty("keycloak.map.storage.connectionsJpa.url", postgres.getJdbcUrl());
+        }
+    }
+
+    private void handleCockroach() {
+        final CockroachContainer cockroach = cockroachdbProvider.getContainer();
+        //if suite container is running stop it
+        if (cockroach != null && cockroach.isRunning()) {
+            cockroach.stop();
+        
+            //it needs some time to stop
+            Retry.executeWithBackoff(i -> {
+                if (cockroach.isRunning()) { 
+                    throw new AssertionError(String.format("Stopping CockroachDB container was not successful. Number of attempts: %d", i));
+                }
+            }, Duration.ofMinutes(1), 50);
+        }
+
+        // spawn new container
+        if (Boolean.parseBoolean(System.getProperty("cockroachdb.start-container-for-zero-downtime", "true"))) {
+            CockroachContainer newCockroach = cockroachdbProvider.createContainer();
+            newCockroach.withStartupAttempts(3); // with podman it sometimes fails with com.sun.jna.LastErrorException: https://github.com/testcontainers/testcontainers-java/issues/6640
+            newCockroach.start();
+
+            log.infof("DatabaseInfo: %s, user=%s, pass=%s", newCockroach.getJdbcUrl(), CockroachdbContainerProvider.COCKROACHDB_DB_USER, CockroachdbContainerProvider.COCKROACHDB_DB_PASSWORD);
+
+            sysProp = new SetSystemProperty("keycloak.map.storage.connectionsJpa.url", newCockroach.getJdbcUrl());
+
+           /* 
+            * CRDB container fails to connect intermitently with: 
+            * org.postgresql.util.PSQLException: FATAL: password authentication failed for user "keycloak"
+            *
+            * It seems it needs some time to set correct permission for a user after fresh start
+            * Waiting for successful connection.
+            */
+            Retry.executeWithBackoff(i -> {
+                try {
+                    DriverManager.getConnection(newCockroach.getJdbcUrl(), CockroachdbContainerProvider.COCKROACHDB_DB_USER, CockroachdbContainerProvider.COCKROACHDB_DB_PASSWORD);
+                } catch (SQLException e) {
+                    throw new AssertionError(String.format("Establishing connection was not successful. Number of attempts: %d", i));
+                }
+            }, Duration.ofMinutes(1), 500);
+        }
+    }
+
+    private void handleHotRod() {
+        InfinispanContainer hotRodContainer = hotrodProvider.getContainer();
+        //if suite container is running stop it
+        if (hotRodContainer != null && hotRodContainer.isRunning()) {
+            hotRodContainer.stop();
+        }
+        // spawn new container
+        if (Boolean.parseBoolean(System.getProperty("start-hotrod-container-for-zero-downtime", "true"))) {
+            hotRodContainer = new InfinispanContainer();
+            hotRodContainer.start();
+
+            sysProp = new SetSystemProperty(HotRodContainerProvider.HOT_ROD_STORE_HOST_PROPERTY, hotRodContainer.getHost());
+        }
+    }
+
+    @Test
+    public void realmSmokeTest() throws Exception {
+        //realm
+        RealmRepresentation testRealmRep = adminClient.realm(AuthRealm.TEST).toRepresentation();
+        String realmAttribute = testRealmRep.getAttributes().get("attr");
+        assertThat(realmAttribute, equalTo("val"));
+        adminClient.realm(AuthRealm.TEST).update(RealmBuilder.edit(testRealmRep).displayName("displayName").build());
+
+        //verify objects can be read by legacy server
+        RealmRepresentation returnedRealmRep = legacyAdminClient.realm(AuthRealm.TEST).toRepresentation();
+        assertThat(returnedRealmRep.getDisplayName(), is("displayName"));
+        assertThat(returnedRealmRep.getAttributes().get("attr"), is("val"));
+    }
+
+    @Test
+    public void clientSmokeTest () throws Exception {
+        //client
+        List<ClientRepresentation> clients = adminClient.realm(AuthRealm.TEST).clients().findByClientId("client");
+        assertThat(clients, hasSize(1));
+        ClientRepresentation clientRep = clients.get(0);
+        adminClient.realm(AuthRealm.TEST).clients().get(clientRep.getId()).update(ClientBuilder.edit(clientRep).addRedirectUri("*").build());
+
+        //client role
+        List<RoleRepresentation> clientRoles = adminClient.realm(AuthRealm.TEST).clients().get(clientRep.getId()).roles().list();
+        assertThat(clientRoles, hasSize(1));
+        RoleRepresentation clientRoleRep = clientRoles.get(0);
+        assertThat(clientRoleRep.getName(), is("client_role"));
+        adminClient.realm(AuthRealm.TEST).clients().get(clientRep.getId()).roles().get(clientRoleRep.getName()).update(RoleBuilder.edit(clientRoleRep).description("desc").build());
+
+        //verify objects can be read by legacy server
+        ClientRepresentation returnedClientRep = legacyAdminClient.realm(AuthRealm.TEST).clients().get(clientRep.getId()).toRepresentation();
+        assertThat(returnedClientRep.getRedirectUris(), hasItem("*"));
+
+        RoleRepresentation returnedClientRoleRep = legacyAdminClient.realm(AuthRealm.TEST).clients().get(clientRep.getId()).roles().get(clientRoleRep.getName()).toRepresentation();
+        assertThat(returnedClientRoleRep.getDescription(), is("desc"));
+        assertThat(returnedClientRoleRep.getAttributes().get("attr"), hasItem("val"));
+    }
+
+    @Test
+    public void clientScopeSmokeTest () throws Exception {
+        //client scope
+        ClientScopeRepresentation clientScopeRep = adminClient.realm(AuthRealm.TEST).clientScopes().findAll().stream().filter(scope -> scope.getName().equals("client_scope")).findFirst().orElse(null);
+        assertThat(clientScopeRep, notNullValue());
+        adminClient.realm(AuthRealm.TEST).clientScopes().get(clientScopeRep.getId()).update(ClientScopeBuilder.edit(clientScopeRep).description("desc").build());
+
+        //verify objects can be read by legacy server
+        ClientScopeRepresentation returnedClientScopeRep = legacyAdminClient.realm(AuthRealm.TEST).clientScopes().get(clientScopeRep.getId()).toRepresentation();
+        assertThat(returnedClientScopeRep.getDescription(), is("desc"));
+    }
+
+    @Test
+    public void userRoleSmokeTest() throws Exception {
+        //user
+        List<UserRepresentation> users = adminClient.realm(AuthRealm.TEST).users().searchByUsername("user", Boolean.TRUE);
+        assertThat(users, hasSize(1));
+        UserRepresentation userRep = users.get(0);
+        adminClient.realm(AuthRealm.TEST).users().get(userRep.getId()).update(UserBuilder.edit(userRep).email("test@test").build());
+
+        //verify objects can be read by legacy server
+        UserRepresentation returnedUserRep = legacyAdminClient.realm(AuthRealm.TEST).users().get(userRep.getId()).toRepresentation();
+        assertThat(returnedUserRep.getEmail(), is("test@test"));
+        assertThat(returnedUserRep.getAttributes().get("attr"), hasItem("val"));
+    }
+
+    @Test
+    public void roleSmokeTest() throws Exception {
+        //realm role
+        RoleRepresentation realmRoleRep = adminClient.realm(AuthRealm.TEST).roles().get("realm_role").toRepresentation();
+        adminClient.realm(AuthRealm.TEST).roles().get("realm_role").update(RoleBuilder.edit(realmRoleRep).description("desc").build());
+
+        //verify objects can be read by legacy server
+        RoleRepresentation returnedRoleRep = legacyAdminClient.realm(AuthRealm.TEST).roles().get(realmRoleRep.getName()).toRepresentation();
+        assertThat(returnedRoleRep.getDescription(), is("desc"));
+        assertThat(returnedRoleRep.getAttributes().get("attr"), hasItem("val"));
+    }
+
+    @Test
+    public void groupSmokeTest() throws Exception {
+        //group
+        List<GroupRepresentation> groups = adminClient.realm(AuthRealm.TEST).groups().groups();
+        assertThat(groups, hasSize(1));
+        GroupRepresentation groupRep = groups.get(0);
+        adminClient.realm(AuthRealm.TEST).groups().group(groupRep.getId()).update(GroupBuilder.edit(groupRep).singleAttribute("attr", "val").build());
+
+        //verify objects can be read by legacy server
+        GroupRepresentation returnedGroupRep = legacyAdminClient.realm(AuthRealm.TEST).groups().group(groupRep.getId()).toRepresentation();
+        assertThat(returnedGroupRep.getAttributes().get("attr"), hasItem("val"));
+    }
+
+    @Override
+    public void addTestRealms(List<RealmRepresentation> testRealms) {
+    }
+}

testsuite/integration-arquillian/tests/base/testsuites/base-suite

@@ -45,3 +45,4 @@ validation,6
 vault,4
 welcomepage,6
 x509,4
+zerodowntime,1

testsuite/integration-arquillian/tests/base/testsuites/database-suite

@@ -17,3 +17,4 @@ TransactionsTest
 UserProfileTest
 org.keycloak.testsuite.admin.**
 org.keycloak.testsuite.authz.**ManagementTest
+org.keycloak.testsuite.zerodowntime.**