Improving quarkus testsuite execution time

Closes #13544

.github/workflows/ci.yml

@@ -474,7 +474,7 @@ jobs:
 
       - name: Run Quarkus Tests in Docker
         run: |
-          ./mvnw clean install -nsu -B -f quarkus/tests/pom.xml -Dkc.quarkus.tests.dist=docker | misc/log/trimmer.sh
+          ./mvnw clean install -nsu -B -f quarkus/tests/pom.xml -Dkc.quarkus.tests.dist=docker -Dtest=StartCommandDistTest | misc/log/trimmer.sh
           TEST_RESULT=${PIPESTATUS[0]}
           exit $TEST_RESULT
 

quarkus/tests/integration/src/main/java/org/keycloak/it/junit5/extension/CLITestExtension.java

@@ -23,6 +23,7 @@ import static org.keycloak.quarkus.runtime.Environment.forceTestLaunchMode;
 import static org.keycloak.quarkus.runtime.cli.command.Main.CONFIG_FILE_LONG_NAME;
 import static org.keycloak.quarkus.runtime.cli.command.Main.CONFIG_FILE_SHORT_NAME;
 
+import java.io.IOException;
 import java.lang.reflect.Method;
 import java.nio.file.Path;
 import java.util.Arrays;
@@ -32,6 +33,7 @@ import java.util.List;
 import java.util.Set;
 import java.util.regex.Pattern;
 
+import io.quarkus.deployment.util.FileUtil;
 import io.quarkus.runtime.configuration.QuarkusConfigFactory;
 import org.junit.jupiter.api.extension.ExtensionContext;
 import org.junit.jupiter.api.extension.ParameterContext;
@@ -91,7 +93,7 @@ public class CLITestExtension extends QuarkusMainTestExtension {
             onKeepServerAlive(context.getRequiredTestMethod().getAnnotation(KeepServerAlive.class));
 
             if (dist == null) {
-                dist = createDistribution(distConfig);
+                dist = createDistribution(distConfig, getLegacyStoreConfig(context), getDatabaseConfig(context));
             }
 
             copyTestProvider(context.getRequiredTestClass().getAnnotation(TestProvider.class));
@@ -100,7 +102,7 @@ public class CLITestExtension extends QuarkusMainTestExtension {
             onBeforeStartDistribution(context.getRequiredTestMethod().getAnnotation(BeforeStartDistribution.class));
 
             if (launch != null) {
-                result = dist.run(Arrays.asList(launch.value()));
+                result = dist.run(List.of(launch.value()));
             }
         } else {
             configureProfile(context);
@@ -108,14 +110,18 @@ public class CLITestExtension extends QuarkusMainTestExtension {
         }
     }
 
+    private static LegacyStore getLegacyStoreConfig(ExtensionContext context) {
+        return context.getTestClass().get().getDeclaredAnnotation(LegacyStore.class);
+    }
+
     private void copyTestProvider(TestProvider provider) {
         if (provider == null) {
             return;
         }
 
-        if (dist instanceof RawKeycloakDistribution) {
+        if (RAW.equals(DistributionType.getCurrent().orElse(RAW))) {
             try {
-                ((RawKeycloakDistribution) dist).copyProvider(provider.value().getDeclaredConstructor().newInstance());
+                dist.unwrap(RawKeycloakDistribution.class).copyProvider(provider.value().getDeclaredConstructor().newInstance());
             } catch (Exception cause) {
                 throw new RuntimeException("Failed to instantiate test provider: " + provider.getClass(), cause);
             }
@@ -167,10 +173,10 @@ public class CLITestExtension extends QuarkusMainTestExtension {
         }
 
         super.afterEach(context);
-        reset();
+        reset(distConfig);
     }
 
-    private void reset() {
+    private void reset(DistributionTest distConfig) {
         QuarkusConfigFactory.setConfig(null);
         //remove the config file property if set, and also the profile, to not have side effects in other tests.
         System.getProperties().remove(Environment.PROFILE);
@@ -184,6 +190,20 @@ public class CLITestExtension extends QuarkusMainTestExtension {
             databaseContainer = null;
         }
         result = null;
+        if (RAW.equals(DistributionType.getCurrent().orElse(RAW))) {
+            if (distConfig != null && !DistributionTest.ReInstall.NEVER.equals(distConfig.reInstall()) && dist != null) {
+                try {
+                    FileUtil.deleteDirectory(getDistPath().getDistRootPath().resolve("conf"));
+                    getDistPath().getDistRootPath().resolve("conf").toFile().mkdirs();
+                    FileUtil.deleteDirectory(getDistPath().getDistRootPath().resolve("providers"));
+                    getDistPath().getDistRootPath().resolve("providers").toFile().mkdirs();
+                    FileUtil.deleteDirectory(getDistPath().getDistRootPath().resolve("data"));
+                    getDistPath().getDistRootPath().resolve("data").toFile().mkdirs();
+                } catch (IOException e) {
+                    throw new RuntimeException("Failed to delete conf directory");
+                }
+            }
+        }
     }
 
     @Override
@@ -192,7 +212,7 @@ public class CLITestExtension extends QuarkusMainTestExtension {
 
         if (distConfig != null) {
             if (BEFORE_ALL.equals(distConfig.reInstall())) {
-                dist = createDistribution(distConfig);
+                dist = createDistribution(distConfig, getLegacyStoreConfig(context), getDatabaseConfig(context));
             }
         } else {
             forceTestLaunchMode();
@@ -210,8 +230,8 @@ public class CLITestExtension extends QuarkusMainTestExtension {
         super.afterAll(context);
     }
 
-    private KeycloakDistribution createDistribution(DistributionTest config) {
-        return DistributionType.getCurrent().orElse(RAW).newInstance(config);
+    private KeycloakDistribution createDistribution(DistributionTest config, LegacyStore legacyStoreConfig, WithDatabase databaseConfig) {
+        return new KeycloakDistributionDecorator(legacyStoreConfig, databaseConfig, DistributionType.getCurrent().orElse(RAW).newInstance(config));
     }
 
     @Override
@@ -240,7 +260,7 @@ public class CLITestExtension extends QuarkusMainTestExtension {
         }
 
         if (type.equals(KeycloakDistribution.class)) {
-            if (dist == null) {
+            if (context.getTestClass().orElse(Object.class).getDeclaredAnnotation(DistributionTest.class) == null) {
                 throw new RuntimeException("Only tests annotated with " + DistributionTest.class + " can inject a distribution instance");
             }
             return dist;
@@ -271,7 +291,7 @@ public class CLITestExtension extends QuarkusMainTestExtension {
     }
 
     private void configureDatabase(ExtensionContext context) {
-        WithDatabase database = context.getTestClass().orElse(Object.class).getDeclaredAnnotation(WithDatabase.class);
+        WithDatabase database = getDatabaseConfig(context);
 
         if (database != null) {
             if (dist == null) {
@@ -301,6 +321,10 @@ public class CLITestExtension extends QuarkusMainTestExtension {
         }
     }
 
+    private static WithDatabase getDatabaseConfig(ExtensionContext context) {
+        return context.getTestClass().orElse(Object.class).getDeclaredAnnotation(WithDatabase.class);
+    }
+
     private void configureDevServices() {
         setProperty("quarkus.vault.devservices.enabled", Boolean.FALSE.toString());
         setProperty("quarkus.datasource.devservices.enabled", Boolean.TRUE.toString());
@@ -327,7 +351,6 @@ public class CLITestExtension extends QuarkusMainTestExtension {
     }
 
     private RawDistRootPath getDistPath(){
-        Path distPath = ((RawKeycloakDistribution)dist).getDistPath();
-        return new RawDistRootPath(distPath);
+        return new RawDistRootPath(dist.unwrap(RawKeycloakDistribution.class).getDistPath());
     }
 }

quarkus/tests/integration/src/main/java/org/keycloak/it/junit5/extension/DistributionTest.java

@@ -30,10 +30,23 @@ public @interface DistributionTest {
 
     boolean debug() default false;
     boolean keepAlive() default false;
+    boolean createAdminUser() default false;
 
     enum ReInstall {
+
+        /**
+         * Install the distribution only once before running a test class.
+         */
         BEFORE_ALL,
+
+        /**
+         * Re-install the distribution before running a test method.
+         */
         BEFORE_TEST,
+
+        /**
+         * Does not reset the distribution such as removing data, providers, and conf directories.
+         */
         NEVER;
     }
 

quarkus/tests/integration/src/main/java/org/keycloak/it/junit5/extension/DistributionType.java

@@ -41,7 +41,8 @@ public enum DistributionType {
                 config.debug(),
                 config.keepAlive(),
                 !DistributionTest.ReInstall.NEVER.equals(config.reInstall()),
-                config.removeBuildOptionsAfterBuild());
+                config.removeBuildOptionsAfterBuild(),
+                config.createAdminUser());
     }
 
     private final Function<DistributionTest, KeycloakDistribution> factory;

quarkus/tests/integration/src/main/java/org/keycloak/it/junit5/extension/KeycloakDistributionDecorator.java

@@ -0,0 +1,128 @@
+/*
+ * Copyright 2022 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.it.junit5.extension;
+
+import java.nio.file.Path;
+import java.util.List;
+import org.keycloak.it.utils.KeycloakDistribution;
+
+public class KeycloakDistributionDecorator implements KeycloakDistribution {
+
+    private LegacyStore legacyStoreConfig;
+    private WithDatabase databaseConfig;
+    private KeycloakDistribution delegate;
+
+    public KeycloakDistributionDecorator(LegacyStore legacyStoreConfig, WithDatabase databaseConfig, KeycloakDistribution delegate) {
+        this.legacyStoreConfig = legacyStoreConfig;
+        this.databaseConfig = databaseConfig;
+        this.delegate = delegate;
+    }
+
+    @Override
+    public CLIResult run(List<String> arguments) {
+        return delegate.run(new ServerOptions(legacyStoreConfig, databaseConfig, arguments));
+    }
+
+    @Override
+    public void stop() {
+        delegate.stop();
+    }
+
+    @Override
+    public List<String> getOutputStream() {
+        return delegate.getOutputStream();
+    }
+
+    @Override
+    public List<String> getErrorStream() {
+        return delegate.getErrorStream();
+    }
+
+    @Override
+    public int getExitCode() {
+        return delegate.getExitCode();
+    }
+
+    @Override
+    public boolean isDebug() {
+        return delegate.isDebug();
+    }
+
+    @Override
+    public boolean isManualStop() {
+        return delegate.isManualStop();
+    }
+
+    @Override
+    public String[] getCliArgs(List<String> arguments) {
+        return delegate.getCliArgs(arguments);
+    }
+
+    @Override
+    public void setManualStop(boolean manualStop) {
+        delegate.setManualStop(manualStop);
+    }
+
+    @Override
+    public void setQuarkusProperty(String key, String value) {
+        delegate.setQuarkusProperty(key, value);
+    }
+
+    @Override
+    public void setProperty(String key, String value) {
+        delegate.setProperty(key, value);
+    }
+
+    @Override
+    public void deleteQuarkusProperties() {
+        delegate.deleteQuarkusProperties();
+    }
+
+    @Override
+    public void copyOrReplaceFileFromClasspath(String file, Path distDir) {
+        delegate.copyOrReplaceFileFromClasspath(file, distDir);
+    }
+
+    @Override
+    public void removeProperty(String name) {
+        delegate.removeProperty(name);
+    }
+
+    @Override
+    public void setEnvVar(String name, String value) {
+        delegate.setEnvVar(name, value);
+    }
+
+    @Override
+    public void copyOrReplaceFile(Path file, Path targetFile) {
+        delegate.copyOrReplaceFile(file, targetFile);
+    }
+
+    @Override
+    public  <D extends KeycloakDistribution> D unwrap(Class<D> type) {
+        if (!KeycloakDistribution.class.isAssignableFrom(type)) {
+            throw new IllegalArgumentException("Not a " + KeycloakDistribution.class + " type");
+        }
+
+        if (type.isInstance(delegate)) {
+            return (D) delegate;
+        }
+
+        throw new IllegalArgumentException("Not a " + type + " type");
+    }
+}

quarkus/tests/integration/src/main/java/org/keycloak/it/junit5/extension/LegacyStore.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2021 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.it.junit5.extension;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+/**
+ * Use this annotation to enable the legacy store when running a test.
+ */
+@Target(ElementType.TYPE)
+@ExtendWith({ CLITestExtension.class })
+@Retention(RetentionPolicy.RUNTIME)
+public @interface LegacyStore {
+
+    /**
+     * If {@code true}, the cache is set to local by default.
+     */
+    boolean defaultLocalCache() default true;
+}

quarkus/tests/integration/src/main/java/org/keycloak/it/junit5/extension/ServerOptions.java

@@ -0,0 +1,87 @@
+/*
+ * Copyright 2022 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.it.junit5.extension;
+
+import static org.keycloak.quarkus.runtime.cli.command.AbstractStartCommand.OPTIMIZED_BUILD_OPTION_LONG;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Predicate;
+import org.keycloak.quarkus.runtime.cli.command.Export;
+import org.keycloak.quarkus.runtime.cli.command.Import;
+import org.keycloak.quarkus.runtime.cli.command.ShowConfig;
+
+final class ServerOptions extends ArrayList<String> {
+
+    private static final Predicate<String> IGNORED_ARGUMENTS = ((Predicate<String>) s -> false)
+            .or(OPTIMIZED_BUILD_OPTION_LONG::equals)
+            .or(Export.NAME::equals)
+            .or(Import.NAME::equals)
+            .or("--help"::equals)
+            .or("--help-all"::equals)
+            .or("-h"::equals)
+            .or(ShowConfig.NAME::equals);
+
+    ServerOptions(LegacyStore legacyStoreConfig, WithDatabase withDatabase, List<String> rawOptions) {
+        if (rawOptions.isEmpty()) {
+            return;
+        }
+
+        for (Map.Entry<String, Predicate<String>> entry : getDefaultOptions(legacyStoreConfig, withDatabase).entrySet()) {
+            if (contains(entry.getKey())) {
+                continue;
+            }
+
+            if (!rawOptions.stream().anyMatch(entry.getValue())) {
+                add(entry.getKey());
+            }
+        }
+
+        addAll(0, rawOptions);
+    }
+
+    private Map<String, Predicate<String>> getDefaultOptions(LegacyStore legacyStoreConfig, WithDatabase withDatabase) {
+        Map<String, Predicate<String>> defaultOptions = new HashMap<>();
+
+        defaultOptions.put("--storage=chm", ignoreStorageChm(legacyStoreConfig, withDatabase));
+        defaultOptions.put("--cache=local", ignoreCacheLocal(legacyStoreConfig));
+
+        return defaultOptions;
+    }
+
+    private Predicate<String> ignoreCacheLocal(LegacyStore legacyStoreConfig) {
+        return new Predicate<String>() {
+            @Override
+            public boolean test(String arg) {
+                return arg.contains("--cache") || legacyStoreConfig == null || !legacyStoreConfig.defaultLocalCache();
+            }
+        }.or(IGNORED_ARGUMENTS);
+    }
+
+    private Predicate<String> ignoreStorageChm(LegacyStore legacyStoreConfig, WithDatabase withDatabase) {
+        return new Predicate<String>() {
+            @Override
+            public boolean test(String arg) {
+                return arg.contains("--storage") || legacyStoreConfig != null || withDatabase != null;
+            }
+        }.or(IGNORED_ARGUMENTS);
+    }
+
+}

quarkus/tests/integration/src/main/java/org/keycloak/it/utils/DockerKeycloakDistribution.java

@@ -203,4 +203,16 @@ public final class DockerKeycloakDistribution implements KeycloakDistribution {
         return this.manualStop;
     }
 
+    @Override
+    public <D extends KeycloakDistribution> D unwrap(Class<D> type) {
+        if (!KeycloakDistribution.class.isAssignableFrom(type)) {
+            throw new IllegalArgumentException("Not a " + KeycloakDistribution.class + " type");
+        }
+
+        if (type.isInstance(this)) {
+            return (D) this;
+        }
+
+        throw new IllegalArgumentException("Not a " + type + " type");
+    }
 }

quarkus/tests/integration/src/main/java/org/keycloak/it/utils/KeycloakDistribution.java

@@ -55,11 +55,13 @@ public interface KeycloakDistribution {
         throw new RuntimeException("Not implemented");
     }
 
-    default void setEnvVar(String kc_db_username, String bad) {
+    default void setEnvVar(String name, String value) {
         throw new RuntimeException("Not implemented");
     }
 
     default void copyOrReplaceFile(Path file, Path targetFile) {
         throw new RuntimeException("Not implemented");
     }
+
+    <D extends KeycloakDistribution> D unwrap(Class<D> type);
 }

quarkus/tests/integration/src/main/java/org/keycloak/it/utils/RawKeycloakDistribution.java

@@ -41,6 +41,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.concurrent.*;
+import java.util.concurrent.locks.LockSupport;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 import javax.net.ssl.HostnameVerifier;
@@ -83,15 +84,18 @@ public final class RawKeycloakDistribution implements KeycloakDistribution {
     private boolean debug;
     private boolean reCreate;
     private boolean removeBuildOptionsAfterBuild;
+    private boolean createAdminUser;
     private ExecutorService outputExecutor;
     private boolean inited = false;
     private Map<String, String> envVars = new HashMap<>();
 
-    public RawKeycloakDistribution(boolean debug, boolean manualStop, boolean reCreate, boolean removeBuildOptionsAfterBuild) {
+    public RawKeycloakDistribution(boolean debug, boolean manualStop, boolean reCreate, boolean removeBuildOptionsAfterBuild,
+            boolean createAdminUser) {
         this.debug = debug;
         this.manualStop = manualStop;
         this.reCreate = reCreate;
         this.removeBuildOptionsAfterBuild = removeBuildOptionsAfterBuild;
+        this.createAdminUser = createAdminUser;
         this.distPath = prepareDistribution();
     }
 
@@ -222,6 +226,8 @@ public final class RawKeycloakDistribution implements KeycloakDistribution {
             allArgs.add("-D" + LAUNCH_MODE + "=test");
         }
 
+        allArgs.add("-Djgroups.join_timeout=50");
+
         this.relativePath = arguments.stream().filter(arg -> arg.startsWith("--http-relative-path")).map(arg -> arg.substring(arg.indexOf('=') + 1)).findAny().orElse("/");
         this.httpPort = Integer.parseInt(arguments.stream().filter(arg -> arg.startsWith("--http-port")).map(arg -> arg.substring(arg.indexOf('=') + 1)).findAny().orElse("8080"));
 
@@ -378,6 +384,9 @@ public final class RawKeycloakDistribution implements KeycloakDistribution {
             while (keycloak.isAlive()) {
                 readStream(outStream, outputStream);
                 readStream(errStream, errorStream);
+                // a hint to temporarily disable the current thread in favor of the process where the distribution is running
+                // after some tests it shows effective to help starting the server faster
+                LockSupport.parkNanos(1L);
             }
         } catch (Throwable cause) {
             throw new RuntimeException("Failed to read server output", cause);
@@ -404,8 +413,10 @@ public final class RawKeycloakDistribution implements KeycloakDistribution {
         ProcessBuilder pb = new ProcessBuilder(getCliArgs(arguments));
         ProcessBuilder builder = pb.directory(distPath.resolve("bin").toFile());
 
+        if (createAdminUser) {
             builder.environment().put("KEYCLOAK_ADMIN", "admin");
             builder.environment().put("KEYCLOAK_ADMIN_PASSWORD", "admin");
+        }
 
         if (debug) {
             builder.environment().put("DEBUG_SUSPEND", "y");
@@ -427,8 +438,8 @@ public final class RawKeycloakDistribution implements KeycloakDistribution {
     }
 
     @Override
-    public void setEnvVar(String key, String value) {
-        this.envVars.put(key, value);
+    public void setEnvVar(String name, String value) {
+        this.envVars.put(name, value);
     }
 
     @Override
@@ -556,4 +567,17 @@ public final class RawKeycloakDistribution implements KeycloakDistribution {
 
         providerJar.as(ZipExporter.class).exportTo(getDistPath().resolve("providers").resolve(providerJar.getName()).toFile());
     }
+
+    @Override
+    public <D extends KeycloakDistribution> D unwrap(Class<D> type) {
+        if (!KeycloakDistribution.class.isAssignableFrom(type)) {
+            throw new IllegalArgumentException("Not a " + KeycloakDistribution.class + " type");
+        }
+
+        if (type.isInstance(this)) {
+            return (D) this;
+        }
+
+        throw new IllegalArgumentException("Not a " + type + " type");
+    }
 }

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/HelpCommandTest.java

@@ -1,106 +0,0 @@
-/*
- * Copyright 2021 Red Hat, Inc. and/or its affiliates
- * and other contributors as indicated by the @author tags.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.keycloak.it.cli;
-
-import org.junit.jupiter.api.Test;
-import org.keycloak.it.junit5.extension.CLIResult;
-import org.keycloak.it.junit5.extension.CLITest;
-import org.keycloak.quarkus.runtime.cli.command.Build;
-
-import io.quarkus.test.junit.main.Launch;
-import io.quarkus.test.junit.main.LaunchResult;
-import org.keycloak.quarkus.runtime.cli.command.Start;
-import org.keycloak.quarkus.runtime.cli.command.StartDev;
-
-import static org.keycloak.quarkus.runtime.cli.command.AbstractStartCommand.OPTIMIZED_BUILD_OPTION_LONG;
-
-@CLITest
-public class HelpCommandTest {
-
-    @Test
-    @Launch({})
-    void testDefaultToHelp(LaunchResult result) {
-        CLIResult cliResult = (CLIResult) result;
-        cliResult.assertHelp();
-    }
-
-    @Test
-    @Launch({ "--help" })
-    void testHelp(LaunchResult result) {
-        CLIResult cliResult = (CLIResult) result;
-        cliResult.assertHelp();
-    }
-
-    @Test
-    @Launch({ "-h" })
-    void testHelpShort(LaunchResult result) {
-        CLIResult cliResult = (CLIResult) result;
-        cliResult.assertHelp();
-    }
-
-    @Test
-    @Launch({ Start.NAME, "--help", OPTIMIZED_BUILD_OPTION_LONG})
-    void testStartOptimizedHelp(LaunchResult result) {
-        CLIResult cliResult = (CLIResult) result;
-        cliResult.assertHelp();
-    }
-
-    @Test
-    @Launch({ Start.NAME, "--help" })
-    void testStartHelp(LaunchResult result) {
-        CLIResult cliResult = (CLIResult) result;
-        cliResult.assertHelp();
-    }
-
-    @Test
-    @Launch({ Start.NAME, "--optimized", "--help-all" })
-    void testStartOptimizedHelpAll(LaunchResult result) {
-        CLIResult cliResult = (CLIResult) result;
-        cliResult.assertHelp();
-        cliResult.assertNoMessage("--storage ");
-    }
-
-    @Test
-    @Launch({ StartDev.NAME, "--help" })
-    void testStartDevHelp(LaunchResult result) {
-        CLIResult cliResult = (CLIResult) result;
-        cliResult.assertHelp();
-    }
-
-    @Test
-    @Launch({ StartDev.NAME, "--help-all" })
-    void testStartDevHelpAll(LaunchResult result) {
-        CLIResult cliResult = (CLIResult) result;
-        cliResult.assertHelp();
-    }
-
-    @Test
-    @Launch({ Start.NAME, "--help-all" })
-    void testStartHelpAll(LaunchResult result) {
-        CLIResult cliResult = (CLIResult) result;
-        cliResult.assertHelp();
-        cliResult.assertMessage("--storage");
-    }
-
-    @Test
-    @Launch({ Build.NAME, "--help" })
-    void testBuildHelp(LaunchResult result) {
-        CLIResult cliResult = (CLIResult) result;
-        cliResult.assertHelp();
-    }
-}

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/LoggingTest.java

@@ -1,41 +0,0 @@
-package org.keycloak.it.cli;
-
-import io.quarkus.test.junit.main.Launch;
-import io.quarkus.test.junit.main.LaunchResult;
-import org.junit.jupiter.api.Test;
-import org.keycloak.it.junit5.extension.CLIResult;
-import org.keycloak.it.junit5.extension.CLITest;
-
-import static org.keycloak.quarkus.runtime.cli.command.Main.CONFIG_FILE_LONG_NAME;
-
-@CLITest
-public class LoggingTest {
-
-    @Test
-    @Launch({ CONFIG_FILE_LONG_NAME+"=src/test/resources/LoggingTest/keycloak.conf", "start-dev" })
-    void failUnknownHandlersInConfFile(LaunchResult result) {
-        CLIResult cliResult = (CLIResult) result;
-        cliResult.assertMessage("Invalid values in list for key: log Values: foo,console. Possible values are a combination of: console,file,gelf");
-    }
-
-    @Test
-    @Launch({ CONFIG_FILE_LONG_NAME+"=src/test/resources/LoggingTest/emptylog.conf", "start-dev" })
-    void failEmptyLogErrorFromConfFileError(LaunchResult result) {
-        CLIResult cliResult = (CLIResult) result;
-        cliResult.assertMessage("Value for configuration key 'log' is empty.");
-    }
-
-    @Test
-    @Launch({ "start-dev","--log=foo,bar" })
-    void failUnknownHandlersInCliCommand(LaunchResult result) {
-        CLIResult cliResult = (CLIResult) result;
-        cliResult.assertError("Invalid value for option '--log': foo,bar");
-    }
-
-    @Test
-    @Launch({ "start-dev","--log=" })
-    void failEmptyLogValueInCliError(LaunchResult result) {
-        CLIResult cliResult = (CLIResult) result;
-        cliResult.assertError("Invalid value for option '--log': .");
-    }
-}

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/StartCommandTest.java

@@ -1,60 +0,0 @@
-/*
- * Copyright 2021 Red Hat, Inc. and/or its affiliates
- * and other contributors as indicated by the @author tags.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.keycloak.it.cli;
-
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.keycloak.quarkus.runtime.cli.command.AbstractStartCommand.OPTIMIZED_BUILD_OPTION_LONG;
-
-import org.junit.jupiter.api.Test;
-import org.keycloak.it.junit5.extension.CLIResult;
-import org.keycloak.it.junit5.extension.CLITest;
-
-import io.quarkus.test.junit.main.Launch;
-import io.quarkus.test.junit.main.LaunchResult;
-
-@CLITest
-public class StartCommandTest {
-
-    @Test
-    @Launch({ "start", "--hostname-strict=false" })
-    void failNoTls(LaunchResult result) {
-        assertTrue(result.getOutput().contains("Key material not provided to setup HTTPS"),
-                () -> "The Output:\n" + result.getOutput() + "doesn't contains the expected string.");
-    }
-
-    @Test
-    @Launch({ "--profile=dev", "start" })
-    void failUsingDevProfile(LaunchResult result) {
-        assertTrue(result.getErrorOutput().contains("ERROR: You can not 'start' the server in development mode. Please re-build the server first, using 'kc.sh build' for the default production mode."),
-                () -> "The Output:\n" + result.getErrorOutput() + "doesn't contains the expected string.");
-    }
-
-    @Test
-    @Launch({ "-v", "start", "--http-enabled=true", "--hostname-strict=false" })
-    void testHttpEnabled(LaunchResult result) {
-        CLIResult cliResult = (CLIResult) result;
-        cliResult.assertStarted();
-    }
-
-    @Test
-    @Launch({ "-v", "start", "--db=dev-mem", OPTIMIZED_BUILD_OPTION_LONG})
-    void failBuildPropertyNotAvailable(LaunchResult result) {
-        CLIResult cliResult = (CLIResult) result;
-        cliResult.assertError("Unknown option: '--db'");
-    }
-}

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/StartDevCommandTest.java

@@ -1,43 +0,0 @@
-/*
- * Copyright 2021 Red Hat, Inc. and/or its affiliates
- * and other contributors as indicated by the @author tags.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.keycloak.it.cli;
-
-import org.junit.jupiter.api.Test;
-import org.keycloak.it.junit5.extension.CLIResult;
-import org.keycloak.it.junit5.extension.CLITest;
-
-import io.quarkus.test.junit.main.Launch;
-import io.quarkus.test.junit.main.LaunchResult;
-
-@CLITest
-public class StartDevCommandTest {
-
-    @Test
-    @Launch({ "start-dev" })
-    void testDevModeWarning(LaunchResult result) {
-        CLIResult cliResult = (CLIResult) result;
-        cliResult.assertStartedDevMode();
-    }
-
-    @Test
-    @Launch({ "start-dev", "--db=dev-mem" })
-    void testBuildPropertyAvailable(LaunchResult result) {
-        CLIResult cliResult = (CLIResult) result;
-        cliResult.assertStartedDevMode();
-    }
-}

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/BuildAndStartDistTest.java

@@ -17,69 +17,50 @@
 
 package org.keycloak.it.cli.dist;
 
-import java.util.function.Consumer;
 import org.junit.jupiter.api.MethodOrderer.OrderAnnotation;
-import org.junit.jupiter.api.Order;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestMethodOrder;
-import org.keycloak.it.junit5.extension.BeforeStartDistribution;
 import org.keycloak.it.junit5.extension.CLIResult;
 import org.keycloak.it.junit5.extension.DistributionTest;
 import org.keycloak.it.junit5.extension.RawDistOnly;
 import org.keycloak.it.utils.KeycloakDistribution;
-
-import io.quarkus.test.junit.main.Launch;
-import io.quarkus.test.junit.main.LaunchResult;
+import org.keycloak.it.utils.RawKeycloakDistribution;
 
 import static org.keycloak.quarkus.runtime.cli.command.AbstractStartCommand.OPTIMIZED_BUILD_OPTION_LONG;
 
-@DistributionTest(reInstall = DistributionTest.ReInstall.NEVER)
+@DistributionTest
 @RawDistOnly(reason = "Containers are immutable")
 @TestMethodOrder(OrderAnnotation.class)
 public class BuildAndStartDistTest {
 
     @Test
-    @Launch({ "build", "--cache=local" })
-    @Order(1)
-    void testBuildWithCliArgs(LaunchResult result) {
-        CLIResult cliResult = (CLIResult) result;
+    void testBuildAndStart(KeycloakDistribution dist) {
+        RawKeycloakDistribution rawDist = dist.unwrap(RawKeycloakDistribution.class);
+        // start using based on the build options set via CLI
+        CLIResult cliResult = rawDist.run("build", "--storage=chm");
         cliResult.assertBuild();
-    }
-
-    @Test
-    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false", OPTIMIZED_BUILD_OPTION_LONG})
-    @Order(2)
-    void testStartUsingCliArgs(LaunchResult result) {
-        CLIResult cliResult = (CLIResult) result;
+        cliResult = rawDist.run("start", "--http-enabled=true", "--hostname-strict=false", OPTIMIZED_BUILD_OPTION_LONG);
+        cliResult.assertNoBuild();
         cliResult.assertStarted();
-        cliResult.assertLocalCache();
-    }
 
-    @Test
-    @BeforeStartDistribution(SetDefaultOptions.class)
-    @Launch({ "build" })
-    @Order(3)
-    void testBuildUsingConfFile(LaunchResult result) {
-        CLIResult cliResult = (CLIResult) result;
+        // start using based on the build options set via conf file
+        rawDist.setProperty("http-enabled", "true");
+        rawDist.setProperty("hostname-strict", "false");
+        rawDist.setProperty("storage", "chm");
+        cliResult = rawDist.run("build");
         cliResult.assertBuild();
-    }
-
-    @Test
-    @Launch({ "start", OPTIMIZED_BUILD_OPTION_LONG})
-    @Order(4)
-    void testStartUsingConfFile(LaunchResult result) {
-        CLIResult cliResult = (CLIResult) result;
+        cliResult = rawDist.run("start", OPTIMIZED_BUILD_OPTION_LONG);
+        cliResult.assertNoBuild();
+        cliResult.assertStarted();
+        // running start without optimized flag should not cause a build
+        cliResult = rawDist.run("start");
+        cliResult.assertNoBuild();
         cliResult.assertStarted();
-        cliResult.assertLocalCache();
-    }
 
-    public static class SetDefaultOptions implements Consumer<KeycloakDistribution> {
-
-        @Override
-        public void accept(KeycloakDistribution distribution) {
-            distribution.setProperty("http-enabled", "true");
-            distribution.setProperty("hostname-strict", "false");
-            distribution.setProperty("cache", "local");
-        }
+        // remove the build option from conf file to force a build during start
+        rawDist.removeProperty("storage");
+        cliResult = rawDist.run("start");
+        cliResult.assertBuild();
+        cliResult.assertStarted();
     }
 }

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/BuildCommandDistTest.java

@@ -19,6 +19,7 @@ package org.keycloak.it.cli.dist;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.keycloak.quarkus.runtime.cli.command.AbstractStartCommand.OPTIMIZED_BUILD_OPTION_LONG;
 
 import org.junit.jupiter.api.Test;
 import org.keycloak.it.junit5.extension.CLIResult;
@@ -72,7 +73,7 @@ class BuildCommandDistTest {
         distribution.run("build", "--cache=local");
         distribution.removeProperty("proxy");
 
-        CLIResult result = distribution.run("start", "--hostname=mykeycloak");
+        CLIResult result = distribution.run("start", "--hostname=mykeycloak", OPTIMIZED_BUILD_OPTION_LONG);
         result.assertMessage("Key material not provided to setup HTTPS");
     }
 }

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/ClusterConfigDistTest.java

@@ -30,6 +30,7 @@ import org.junit.jupiter.api.condition.OS;
 import org.keycloak.it.junit5.extension.BeforeStartDistribution;
 import org.keycloak.it.junit5.extension.CLIResult;
 import org.keycloak.it.junit5.extension.DistributionTest;
+import org.keycloak.it.junit5.extension.LegacyStore;
 import org.keycloak.it.junit5.extension.RawDistOnly;
 import org.keycloak.it.utils.KeycloakDistribution;
 
@@ -38,6 +39,7 @@ import io.quarkus.test.junit.main.LaunchResult;
 
 @DistributionTest(reInstall = DistributionTest.ReInstall.BEFORE_TEST)
 @RawDistOnly(reason = "Not possible to mount files using docker.")
+@LegacyStore(defaultLocalCache = false)
 public class ClusterConfigDistTest {
 
     @Test
@@ -85,7 +87,7 @@ public class ClusterConfigDistTest {
 
     @Test
     @EnabledOnOs(value = { OS.WINDOWS }, disabledReason = "different shell behaviour on Windows.")
-    @Launch({ "start", "--log-level=\"info,org.infinispan.remoting.transport.jgroups.JGroupsTransport:debug","--http-enabled=true\"", "--hostname-strict=false" })
+    @Launch({ "start", "--log-level=\"info,org.infinispan.remoting.transport.jgroups.JGroupsTransport:debug\"","--http-enabled=true", "--hostname-strict=false" })
     void testWinStartDefaultsToClustering(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
         cliResult.assertStarted();

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/CustomJpaUserProviderDistTest.java

@@ -20,6 +20,7 @@ package org.keycloak.it.cli.dist;
 import org.junit.jupiter.api.Test;
 import org.keycloak.it.junit5.extension.CLIResult;
 import org.keycloak.it.junit5.extension.DistributionTest;
+import org.keycloak.it.junit5.extension.LegacyStore;
 import org.keycloak.it.junit5.extension.RawDistOnly;
 import org.keycloak.it.junit5.extension.TestProvider;
 import com.acme.provider.user.CustomUserProvider;
@@ -27,8 +28,9 @@ import com.acme.provider.user.CustomUserProvider;
 import io.quarkus.test.junit.main.Launch;
 import io.quarkus.test.junit.main.LaunchResult;
 
-@DistributionTest(reInstall = DistributionTest.ReInstall.BEFORE_TEST)
+@DistributionTest
 @RawDistOnly(reason = "Containers are immutable")
+@LegacyStore
 public class CustomJpaUserProviderDistTest {
 
     @Test

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/ExportDistTest.java

@@ -21,27 +21,22 @@ import org.junit.jupiter.api.Test;
 import org.keycloak.it.junit5.extension.CLIResult;
 import org.keycloak.it.junit5.extension.DistributionTest;
 import org.keycloak.it.junit5.extension.RawDistOnly;
-
-import io.quarkus.test.junit.main.Launch;
-import io.quarkus.test.junit.main.LaunchResult;
+import org.keycloak.it.utils.KeycloakDistribution;
 
 @RawDistOnly(reason = "Containers are immutable")
 @DistributionTest
 public class ExportDistTest {
 
     @Test
-    @Launch({"export", "--realm=master", "--dir=."})
-    void testExport(LaunchResult result) {
-        CLIResult cliResult = (CLIResult) result;
+    void testExport(KeycloakDistribution dist) {
+        CLIResult cliResult = dist.run("build");
+
+        cliResult = dist.run("export", "--realm=master", "--dir=.");
         cliResult.assertMessage("Export of realm 'master' requested.");
         cliResult.assertMessage("Export finished successfully");
         cliResult.assertNoMessage("Changes detected in configuration");
-    }
 
-    @Test
-    @Launch({"export", "--realm=master" })
-    void testMissingDir(LaunchResult result) {
-        CLIResult cliResult = (CLIResult) result;
+        cliResult = dist.run("export", "--realm=master");
         cliResult.assertError("Must specify either --dir or --file options.");
     }
 }

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/FeaturesDistTest.java

@@ -4,14 +4,15 @@ import io.quarkus.test.junit.main.Launch;
 import io.quarkus.test.junit.main.LaunchResult;
 import org.hamcrest.CoreMatchers;
 import org.junit.jupiter.api.MethodOrderer;
-import org.junit.jupiter.api.Order;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestMethodOrder;
 import org.junit.jupiter.api.condition.EnabledOnOs;
 import org.junit.jupiter.api.condition.OS;
 import org.keycloak.it.junit5.extension.CLIResult;
 import org.keycloak.it.junit5.extension.DistributionTest;
+import org.keycloak.it.junit5.extension.LegacyStore;
 import org.keycloak.it.junit5.extension.RawDistOnly;
+import org.keycloak.it.utils.KeycloakDistribution;
 import org.keycloak.quarkus.runtime.cli.command.Build;
 import org.keycloak.quarkus.runtime.cli.command.Start;
 import org.keycloak.quarkus.runtime.cli.command.StartDev;
@@ -24,22 +25,18 @@ import static org.keycloak.quarkus.runtime.cli.command.AbstractStartCommand.OPTI
 @DistributionTest
 @RawDistOnly(reason = "Containers are immutable")
 @TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+@LegacyStore
 public class FeaturesDistTest {
 
     @Test
-    @Launch({ Build.NAME, "--features=preview", "--cache=local"})
-    @Order(1)
-    public void testEnableOnBuild(LaunchResult result) {
-        CLIResult cliResult = (CLIResult) result;
+    public void testEnableOnBuild(KeycloakDistribution dist) {
+        CLIResult cliResult = dist.run(Build.NAME, "--features=preview");
         cliResult.assertBuild();
         assertPreviewFeaturesEnabled(cliResult);
-    }
 
-    @Test
-    @Launch({ Start.NAME, "--http-enabled=true", "--hostname-strict=false", OPTIMIZED_BUILD_OPTION_LONG})
-    @Order(2)
-    public void testFeatureEnabledOnStart(LaunchResult result) {
-        assertPreviewFeaturesEnabled((CLIResult) result);
+        cliResult = dist.run(Start.NAME, "--http-enabled=true", "--hostname-strict=false", OPTIMIZED_BUILD_OPTION_LONG);
+        assertPreviewFeaturesEnabled(cliResult);
+
     }
 
     @Test

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/FipsDistTest.java

@@ -29,12 +29,12 @@ import org.keycloak.it.utils.RawKeycloakDistribution;
 import io.quarkus.test.junit.main.Launch;
 import io.quarkus.test.junit.main.LaunchResult;
 
-@DistributionTest(reInstall = DistributionTest.ReInstall.BEFORE_TEST)
+@DistributionTest(createAdminUser = true)
 @RawDistOnly(reason = "Containers are immutable")
 public class FipsDistTest {
 
     @Test
-    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false", "--fips-mode=enabled", "--cache=local", "--log-level=org.keycloak.common.crypto.CryptoIntegration:trace" })
+    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false", "--fips-mode=enabled", "--log-level=org.keycloak.common.crypto.CryptoIntegration:trace" })
     @BeforeStartDistribution(FipsDistTest.InstallBcFipsDependencies.class)
     void testFipsNonApprovedMode(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
@@ -44,7 +44,7 @@ public class FipsDistTest {
     }
 
     @Test
-    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false", "--fips-mode=strict", "--cache=local", "--log-level=org.keycloak.common.crypto.CryptoIntegration:trace" })
+    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false", "--fips-mode=strict", "--log-level=org.keycloak.common.crypto.CryptoIntegration:trace" })
     @BeforeStartDistribution(FipsDistTest.InstallBcFipsDependencies.class)
     void testFipsApprovedMode(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
@@ -55,7 +55,7 @@ public class FipsDistTest {
     }
 
     @Test
-    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false", "--fips-mode=enabled", "--cache=local", "--log-level=org.keycloak.common.crypto.CryptoIntegration:trace" })
+    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false", "--fips-mode=enabled", "--log-level=org.keycloak.common.crypto.CryptoIntegration:trace" })
     void failStartDueToMissingFipsDependencies(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
         cliResult.assertError("Failed to configure FIPS. Make sure you have added the Bouncy Castle FIPS dependencies to the 'providers' directory.");
@@ -65,7 +65,7 @@ public class FipsDistTest {
 
         @Override
         public void accept(KeycloakDistribution distribution) {
-            RawKeycloakDistribution rawDist = (RawKeycloakDistribution) distribution;
+            RawKeycloakDistribution rawDist = distribution.unwrap(RawKeycloakDistribution.class);
             rawDist.copyProvider("org.bouncycastle", "bc-fips");
             rawDist.copyProvider("org.bouncycastle", "bctls-fips");
             rawDist.copyProvider("org.bouncycastle", "bcpkix-fips");

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/HelpCommandDistTest.java

@@ -18,19 +18,97 @@
 package org.keycloak.it.cli.dist;
 
 import static org.junit.Assert.assertEquals;
+import static org.keycloak.quarkus.runtime.cli.command.AbstractStartCommand.OPTIMIZED_BUILD_OPTION_LONG;
 
 import java.util.List;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.condition.OS;
-import org.keycloak.it.cli.HelpCommandTest;
 import org.keycloak.it.junit5.extension.CLIResult;
 import org.keycloak.it.junit5.extension.DistributionTest;
 import org.keycloak.it.junit5.extension.RawDistOnly;
 import org.keycloak.it.utils.KeycloakDistribution;
+import org.keycloak.quarkus.runtime.cli.command.Build;
+import org.keycloak.quarkus.runtime.cli.command.Start;
+import org.keycloak.quarkus.runtime.cli.command.StartDev;
+
+import io.quarkus.test.junit.main.Launch;
+import io.quarkus.test.junit.main.LaunchResult;
 
 @DistributionTest
 @RawDistOnly(reason = "Verifying the help message output doesn't need long spin-up of docker dist tests.")
-public class HelpCommandDistTest extends HelpCommandTest {
+public class HelpCommandDistTest {
+
+    @Test
+    @Launch({})
+    void testDefaultToHelp(LaunchResult result) {
+        CLIResult cliResult = (CLIResult) result;
+        cliResult.assertHelp();
+    }
+
+    @Test
+    @Launch({ "--help" })
+    void testHelp(LaunchResult result) {
+        CLIResult cliResult = (CLIResult) result;
+        cliResult.assertHelp();
+    }
+
+    @Test
+    @Launch({ "-h" })
+    void testHelpShort(LaunchResult result) {
+        CLIResult cliResult = (CLIResult) result;
+        cliResult.assertHelp();
+    }
+
+    @Test
+    @Launch({ Start.NAME, "--help", OPTIMIZED_BUILD_OPTION_LONG})
+    void testStartOptimizedHelp(LaunchResult result) {
+        CLIResult cliResult = (CLIResult) result;
+        cliResult.assertHelp();
+    }
+
+    @Test
+    @Launch({ Start.NAME, "--help" })
+    void testStartHelp(LaunchResult result) {
+        CLIResult cliResult = (CLIResult) result;
+        cliResult.assertHelp();
+    }
+
+    @Test
+    @Launch({ Start.NAME, "--optimized", "--help-all" })
+    void testStartOptimizedHelpAll(LaunchResult result) {
+        CLIResult cliResult = (CLIResult) result;
+        cliResult.assertHelp();
+        cliResult.assertNoMessage("--storage ");
+    }
+
+    @Test
+    @Launch({ StartDev.NAME, "--help" })
+    void testStartDevHelp(LaunchResult result) {
+        CLIResult cliResult = (CLIResult) result;
+        cliResult.assertHelp();
+    }
+
+    @Test
+    @Launch({ StartDev.NAME, "--help-all" })
+    void testStartDevHelpAll(LaunchResult result) {
+        CLIResult cliResult = (CLIResult) result;
+        cliResult.assertHelp();
+    }
+
+    @Test
+    @Launch({ Start.NAME, "--help-all" })
+    void testStartHelpAll(LaunchResult result) {
+        CLIResult cliResult = (CLIResult) result;
+        cliResult.assertHelp();
+        cliResult.assertMessage("--storage");
+    }
+
+    @Test
+    @Launch({ Build.NAME, "--help" })
+    void testBuildHelp(LaunchResult result) {
+        CLIResult cliResult = (CLIResult) result;
+        cliResult.assertHelp();
+    }
 
     @Test
     public void testHelpDoesNotStartReAugJvm(KeycloakDistribution dist) {

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/HostnameDistTest.java

@@ -31,7 +31,7 @@ import org.keycloak.protocol.oidc.representations.OIDCConfigurationRepresentatio
 import io.quarkus.test.junit.main.Launch;
 import io.restassured.RestAssured;
 
-@DistributionTest(keepAlive = true, reInstall = DistributionTest.ReInstall.BEFORE_TEST)
+@DistributionTest(keepAlive = true)
 @BeforeStartDistribution(CopyTLSKeystore.class)
 @RawDistOnly(reason = "Containers are immutable")
 public class HostnameDistTest {
@@ -42,7 +42,7 @@ public class HostnameDistTest {
     }
 
     @Test
-    @Launch({ "start-dev", "--hostname=mykeycloak.127.0.0.1.nip.io" })
+    @Launch({ "start", "--hostname=mykeycloak.127.0.0.1.nip.io", "--http-enabled=true", "--hostname-strict-https=false" })
     public void testSchemeAndPortFromRequestWhenNoProxySet() {
         assertFrontEndUrl("http://mykeycloak.127.0.0.1.nip.io:8080", "http://mykeycloak.127.0.0.1.nip.io:8080/");
         assertFrontEndUrl("http://localhost:8080", "http://mykeycloak.127.0.0.1.nip.io:8080/");
@@ -50,59 +50,66 @@ public class HostnameDistTest {
     }
 
     @Test
-    @Launch({ "start-dev", "--hostname=mykeycloak.127.0.0.1.nip.io", "--hostname-strict-https=true" })
+    @Launch({ "start", "--hostname=mykeycloak.127.0.0.1.nip.io", "--http-enabled=true" })
     public void testForceHttpsSchemeAndPortWhenStrictHttpsEnabled() {
         assertFrontEndUrl("http://mykeycloak.127.0.0.1.nip.io:8080", "https://mykeycloak.127.0.0.1.nip.io:8443/");
         assertFrontEndUrl("http://localhost:8080", "https://mykeycloak.127.0.0.1.nip.io:8443/");
     }
 
     @Test
-    @Launch({ "start-dev", "--hostname=mykeycloak.127.0.0.1.nip.io", "--hostname-port=8443" })
+    @Launch({ "start", "--hostname=mykeycloak.127.0.0.1.nip.io", "--hostname-port=8443", "--http-enabled=true", "--hostname-strict-https=false" })
     public void testForceHostnamePortWhenNoProxyIsSet() {
         assertFrontEndUrl("http://mykeycloak.127.0.0.1.nip.io:8080", "http://mykeycloak.127.0.0.1.nip.io:8443/");
         assertFrontEndUrl("https://mykeycloak.127.0.0.1.nip.io:8443", "https://mykeycloak.127.0.0.1.nip.io:8443/");
     }
 
     @Test
-    @Launch({ "start-dev", "--hostname=mykeycloak.127.0.0.1.nip.io", "--proxy=edge" })
+    @Launch({ "start", "--hostname=mykeycloak.127.0.0.1.nip.io", "--proxy=edge" })
     public void testUseDefaultPortsWhenProxyIsSet() {
+        assertFrontEndUrl("http://mykeycloak.127.0.0.1.nip.io:8080", "https://mykeycloak.127.0.0.1.nip.io/");
+        assertFrontEndUrl("https://mykeycloak.127.0.0.1.nip.io:8443", "https://mykeycloak.127.0.0.1.nip.io/");
+    }
+
+    @Test
+    @Launch({ "start", "--hostname=mykeycloak.127.0.0.1.nip.io", "--proxy=edge", "--hostname-strict-https=false" })
+    public void testUseDefaultPortsWhenProxyIsSetNoStrictHttps() {
         assertFrontEndUrl("http://mykeycloak.127.0.0.1.nip.io:8080", "http://mykeycloak.127.0.0.1.nip.io/");
         assertFrontEndUrl("https://mykeycloak.127.0.0.1.nip.io:8443", "https://mykeycloak.127.0.0.1.nip.io/");
     }
 
     @Test
-    @Launch({ "start-dev", "--hostname=mykeycloak.127.0.0.1.nip.io", "--proxy=edge", "--hostname-strict-https=true" })
+    @Launch({ "start", "--hostname=mykeycloak.127.0.0.1.nip.io", "--proxy=edge", "--hostname-strict-https=true" })
     public void testUseDefaultPortsAndHttpsSchemeWhenProxyIsSetAndStrictHttpsEnabled() {
         assertFrontEndUrl("http://mykeycloak.127.0.0.1.nip.io:8080", "https://mykeycloak.127.0.0.1.nip.io/");
     }
 
     @Test
-    @Launch({ "start-dev", "--hostname=mykeycloak.127.0.0.1.nip.io" })
+    @Launch({ "start", "--hostname=mykeycloak.127.0.0.1.nip.io", "--http-enabled=true", "--hostname-strict-https=false" })
     public void testBackEndUrlFromRequest() {
         assertBackEndUrl("http://localhost:8080", "http://localhost:8080/");
     }
 
     @Test
-    @Launch({ "start-dev", "--hostname=mykeycloak.127.0.0.1.nip.io", "--hostname-strict-backchannel=true" })
+    @Launch({ "start", "--hostname=mykeycloak.127.0.0.1.nip.io", "--hostname-strict-backchannel=true", "--http-enabled=true", "--hostname-strict-https=false" })
     public void testBackEndUrlSameAsFrontEndUrl() {
         assertBackEndUrl("http://localhost:8080", "http://mykeycloak.127.0.0.1.nip.io:8080/");
     }
 
     @Test
-    @Launch({ "start-dev", "--hostname=mykeycloak.127.0.0.1.nip.io", "--hostname-path=/auth", "--hostname-strict=true", "--hostname-strict-backchannel=true" })
+    @Launch({ "start", "--hostname=mykeycloak.127.0.0.1.nip.io", "--hostname-path=/auth", "--hostname-strict=true", "--hostname-strict-backchannel=true", "--http-enabled=true", "--hostname-strict-https=false" })
     public void testSetHostnamePath() {
         assertFrontEndUrl("http://localhost:8080", "http://mykeycloak.127.0.0.1.nip.io:8080/auth/");
         assertBackEndUrl("http://localhost:8080", "http://mykeycloak.127.0.0.1.nip.io:8080/auth/");
     }
 
     @Test
-    @Launch({ "start-dev", "--hostname=mykeycloak.127.0.0.1.nip.io", "--https-port=8543", "--hostname-strict-https=true" })
+    @Launch({ "start", "--hostname=mykeycloak.127.0.0.1.nip.io", "--https-port=8543", "--hostname-strict-https=true", "--http-enabled=true" })
     public void testDefaultTlsPortChangeWhenHttpPortSet() {
         assertFrontEndUrl("http://mykeycloak.127.0.0.1.nip.io:8080", "https://mykeycloak.127.0.0.1.nip.io:8543/");
     }
 
     @Test
-    @Launch({ "start-dev", "--hostname=mykeycloak.127.0.0.1.nip.io", "--hostname-strict-https=true", "--hostname-port=8543" })
+    @Launch({ "start", "--hostname=mykeycloak.127.0.0.1.nip.io", "--hostname-strict-https=true", "--hostname-port=8543", "--http-enabled=true" })
     public void testWelcomePageAdminUrl() {
         Assert.assertTrue(when().get("http://mykeycloak.127.0.0.1.nip.io:8080").asString().contains("http://mykeycloak.127.0.0.1.nip.io:8080/admin/"));
         Assert.assertTrue(when().get("https://mykeycloak.127.0.0.1.nip.io:8443").asString().contains("https://mykeycloak.127.0.0.1.nip.io:8443/admin/"));
@@ -111,14 +118,14 @@ public class HostnameDistTest {
     }
 
     @Test
-    @Launch({ "start-dev", "--hostname=mykeycloak.127.0.0.1.nip.io", "--hostname-admin=mykeycloakadmin.127.0.0.1.nip.io" })
+    @Launch({ "start", "--hostname=mykeycloak.127.0.0.1.nip.io", "--hostname-admin=mykeycloakadmin.127.0.0.1.nip.io", "--http-enabled=true" })
     public void testHostnameAdminSet() {
         Assert.assertTrue(when().get("https://mykeycloak.127.0.0.1.nip.io:8443/admin/master/console").asString().contains("\"authUrl\": \"https://mykeycloakadmin.127.0.0.1.nip.io:8443\""));
         Assert.assertTrue(when().get("https://mykeycloak.127.0.0.1.nip.io:8443/realms/master/protocol/openid-connect/auth?client_id=security-admin-console&redirect_uri=https://mykeycloakadmin.127.0.0.1.nip.io:8443/admin/master/console&state=02234324-d91e-4bf2-8396-57498e96b12a&response_mode=fragment&response_type=code&scope=openid&nonce=f8f3812e-e349-4bbf-8d15-cbba4927f5e5&code_challenge=7qjD_v11WGkt1ig-ZFHxJdrEvuTlzjFRgRGQ_5ADcko&code_challenge_method=S256").asString().contains("Sign in to your account"));
     }
 
     @Test
-    @Launch({ "start-dev", "--hostname=mykeycloak.127.0.0.1.nip.io" })
+    @Launch({ "start", "--hostname=mykeycloak.127.0.0.1.nip.io", "--http-enabled=true" })
     public void testInvalidRedirectUriWhenAdminNotSet() {
         Assert.assertTrue(when().get("https://mykeycloak.127.0.0.1.nip.io:8443/realms/master/protocol/openid-connect/auth?client_id=security-admin-console&redirect_uri=https://mykeycloakadmin.127.0.0.1.nip.io:8443/admin/master/console&state=02234324-d91e-4bf2-8396-57498e96b12a&response_mode=fragment&response_type=code&scope=openid&nonce=f8f3812e-e349-4bbf-8d15-cbba4927f5e5&code_challenge=7qjD_v11WGkt1ig-ZFHxJdrEvuTlzjFRgRGQ_5ADcko&code_challenge_method=S256").asString().contains("Invalid parameter: redirect_uri"));
     }

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/ImportAtStartupDistTest.java

@@ -32,7 +32,7 @@ import org.keycloak.it.utils.RawKeycloakDistribution;
 import io.quarkus.test.junit.main.Launch;
 import io.quarkus.test.junit.main.LaunchResult;
 
-@DistributionTest(reInstall = DistributionTest.ReInstall.BEFORE_TEST)
+@DistributionTest
 @RawDistOnly(reason = "Containers are immutable")
 public class ImportAtStartupDistTest {
 
@@ -84,7 +84,7 @@ public class ImportAtStartupDistTest {
         public void accept(KeycloakDistribution distribution) {
             distribution.copyOrReplaceFileFromClasspath("/quickstart-realm.json", Path.of("data", "import", "realm.json"));
 
-            RawKeycloakDistribution rawDist = (RawKeycloakDistribution) distribution;
+            RawKeycloakDistribution rawDist = distribution.unwrap(RawKeycloakDistribution.class);
 
             rawDist.getDistPath().resolve("data").resolve("import").resolve("sub-dir").toFile().mkdirs();
         }

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/ImportDistTest.java

@@ -18,15 +18,12 @@
 package org.keycloak.it.cli.dist;
 
 import org.junit.jupiter.api.MethodOrderer;
-import org.junit.jupiter.api.Order;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestMethodOrder;
 import org.keycloak.it.junit5.extension.CLIResult;
 import org.keycloak.it.junit5.extension.DistributionTest;
 import org.keycloak.it.junit5.extension.RawDistOnly;
-
-import io.quarkus.test.junit.main.Launch;
-import io.quarkus.test.junit.main.LaunchResult;
+import org.keycloak.it.utils.KeycloakDistribution;
 
 @DistributionTest
 @RawDistOnly(reason = "Containers are immutable")
@@ -34,19 +31,14 @@ import io.quarkus.test.junit.main.LaunchResult;
 public class ImportDistTest {
 
     @Test
-    @Order(1)
-    @Launch({"export", "--realm=master", "--dir=."})
-    void testExport(LaunchResult result) {
-        CLIResult cliResult = (CLIResult) result;
+    void testImport(KeycloakDistribution dist) {
+        CLIResult cliResult = dist.run("build");
+
+        cliResult = dist.run("export", "--realm=master", "--dir=.");
         cliResult.assertMessage("Export of realm 'master' requested.");
         cliResult.assertMessage("Export finished successfully");
-    }
 
-    @Test
-    @Order(2)
-    @Launch({"import", "--dir=." })
-    void testMissingDir(LaunchResult result) {
-        CLIResult cliResult = (CLIResult) result;
+        cliResult = dist.run("import", "--dir=.");
         cliResult.assertMessage("Realm 'master' imported");
         cliResult.assertMessage("Import finished successfully");
         cliResult.assertNoMessage("Changes detected in configuration");

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/LoggingDistTest.java

@@ -19,6 +19,7 @@ package org.keycloak.it.cli.dist;
 
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.keycloak.quarkus.runtime.cli.command.Main.CONFIG_FILE_LONG_NAME;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import org.junit.jupiter.api.Test;
@@ -31,8 +32,9 @@ import org.keycloak.it.junit5.extension.RawDistOnly;
 
 import io.quarkus.test.junit.main.Launch;
 import io.quarkus.test.junit.main.LaunchResult;
+
+import org.keycloak.it.utils.KeycloakDistribution;
 import org.keycloak.it.utils.RawDistRootPath;
-import org.keycloak.quarkus.runtime.configuration.mappers.LoggingPropertyMappers;
 import org.testcontainers.shaded.org.apache.commons.io.FileUtils;
 
 import java.io.File;
@@ -41,7 +43,7 @@ import java.nio.charset.Charset;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 
-@DistributionTest(reInstall = DistributionTest.ReInstall.NEVER)
+@DistributionTest
 @RawDistOnly(reason = "Too verbose for docker and enough to check raw dist")
 public class LoggingDistTest {
 
@@ -64,22 +66,20 @@ public class LoggingDistTest {
 
     @Test
     @EnabledOnOs(value = { OS.LINUX, OS.MAC }, disabledReason = "different shell escaping behaviour on Windows.")
-    @Launch({ "start-dev", "--log-level=off,org.keycloak:debug,org.infinispan:info" })
+    @Launch({ "start-dev", "--log-level=off,org.keycloak:debug" })
     void testRootAndCategoryLevels(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
         assertFalse(cliResult.getOutput().contains("INFO  [io.quarkus"));
         assertTrue(cliResult.getOutput().contains("DEBUG [org.keycloak"));
-        assertTrue(cliResult.getOutput().contains("INFO  [org.infinispan.CONTAINER]"));
     }
 
     @Test
     @EnabledOnOs(value = { OS.WINDOWS }, disabledReason = "different shell escaping behaviour on Windows.")
-    @Launch({ "start-dev", "--log-level=\"off,org.keycloak:debug,org.infinispan:info\"" })
+    @Launch({ "start-dev", "--log-level=\"off,org.keycloak:debug\"" })
     void testWinRootAndCategoryLevels(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
         assertFalse(cliResult.getOutput().contains("INFO  [io.quarkus"));
         assertTrue(cliResult.getOutput().contains("DEBUG [org.keycloak"));
-        assertTrue(cliResult.getOutput().contains("INFO  [org.infinispan.CONTAINER]"));
     }
 
     @Test
@@ -97,7 +97,7 @@ public class LoggingDistTest {
     @Launch({ "start-dev", "--log-level=\"off,org.keycloak:warn,debug\"" })
     void testWinSetLastRootLevelIfMultipleSet(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
-        assertTrue(cliResult.getOutput().contains("DEBUG [org.hibernate"));
+        assertTrue(cliResult.getOutput().contains("DEBUG [io.quarkus.resteasy.runtime]"));
         assertFalse(cliResult.getOutput().contains("INFO  [org.keycloak"));
         cliResult.assertStartedDevMode();
     }
@@ -121,23 +121,23 @@ public class LoggingDistTest {
 
     @Test
     @EnabledOnOs(value = { OS.LINUX, OS.MAC }, disabledReason = "different shell escaping behaviour on Windows.")
-    @Launch({ "start-dev", "--log-level=off,org.keycloak:debug,org.infinispan:info", "--log-console-output=json" })
+    @Launch({ "start-dev", "--log-level=off,org.keycloak:debug,liquibase:debug", "--log-console-output=json" })
     void testLogLevelSettingsAppliedWhenJsonEnabled(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
         assertFalse(cliResult.getOutput().contains("\"loggerName\":\"io.quarkus\",\"level\":\"INFO\")"));
         assertTrue(cliResult.getOutput().contains("\"loggerName\":\"org.keycloak.services.resources.KeycloakApplication\",\"level\":\"DEBUG\""));
-        assertTrue(cliResult.getOutput().contains("\"loggerName\":\"org.infinispan.CONTAINER\",\"level\":\"INFO\""));
+        assertTrue(cliResult.getOutput().contains("\"loggerName\":\"liquibase.servicelocator\",\"level\":\"FINE\""));
     }
 
     @Test
     @EnabledOnOs(value = { OS.WINDOWS }, disabledReason = "different shell escaping behaviour on Windows.")
-    @Launch({ "start-dev", "--log-level=\"off,org.keycloak:debug,org.infinispan:info\"", "--log-console-output=json" })
+    @Launch({ "start-dev", "--log-level=\"off,org.keycloak:debug,liquibase:debug\"", "--log-console-output=json" })
     void testWinLogLevelSettingsAppliedWhenJsonEnabled(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
         assertFalse(cliResult.getOutput().contains("\"loggerName\":\"io.quarkus\",\"level\":\"INFO\")"));
-        assertTrue(cliResult.getOutput().contains("\"loggerName\":\"org.keycloak.quarkus.runtime.storage.legacy.database.LegacyJpaConnectionProviderFactory\",\"level\":\"DEBUG\""));
         assertTrue(cliResult.getOutput().contains("\"loggerName\":\"org.keycloak.services.resources.KeycloakApplication\",\"level\":\"DEBUG\""));
-        assertTrue(cliResult.getOutput().contains("\"loggerName\":\"org.infinispan.CONTAINER\",\"level\":\"INFO\""));
+        assertTrue(cliResult.getOutput().contains("\"loggerName\":\"org.keycloak.protocol.oidc.OIDCWellKnownProviderFactory\",\"level\":\"DEBUG\""));
+        assertTrue(cliResult.getOutput().contains("\"loggerName\":\"liquibase.servicelocator\",\"level\":\"FINE\""));
     }
 
     @Test
@@ -175,4 +175,32 @@ public class LoggingDistTest {
         CLIResult cliResult = (CLIResult) result;
         assertFalse(cliResult.getOutput().contains("INFO  [io.quarkus]"));
     }
+
+    @Test
+    void failUnknownHandlersInConfFile(KeycloakDistribution dist) {
+        dist.copyOrReplaceFileFromClasspath("/logging/keycloak.conf", Paths.get("conf", "keycloak.conf"));
+        CLIResult cliResult = dist.run("start-dev");
+        cliResult.assertMessage("Invalid values in list for key: log Values: foo,console. Possible values are a combination of: console,file,gelf");
+    }
+
+    @Test
+    void failEmptyLogErrorFromConfFileError(KeycloakDistribution dist) {
+        dist.copyOrReplaceFileFromClasspath("/logging/emptylog.conf", Paths.get("conf", "emptylog.conf"));
+        CLIResult cliResult = dist.run(CONFIG_FILE_LONG_NAME+"=../conf/emptylog.conf", "start-dev");
+        cliResult.assertMessage("Value for configuration key 'log' is empty.");
+    }
+
+    @Test
+    @Launch({ "start-dev","--log=foo,bar" })
+    void failUnknownHandlersInCliCommand(LaunchResult result) {
+        CLIResult cliResult = (CLIResult) result;
+        cliResult.assertError("Invalid value for option '--log': foo,bar");
+    }
+
+    @Test
+    @Launch({ "start-dev","--log=" })
+    void failEmptyLogValueInCliError(LaunchResult result) {
+        CLIResult cliResult = (CLIResult) result;
+        cliResult.assertError("Invalid value for option '--log': .");
+    }
 }

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/ProxyDistTest.java

@@ -33,7 +33,7 @@ import org.keycloak.protocol.oidc.representations.OIDCConfigurationRepresentatio
 import io.quarkus.test.junit.main.Launch;
 import io.restassured.RestAssured;
 
-@DistributionTest(keepAlive = true, reInstall = DistributionTest.ReInstall.BEFORE_TEST)
+@DistributionTest(keepAlive = true)
 @BeforeStartDistribution(CopyTLSKeystore.class)
 @RawDistOnly(reason = "Containers are immutable")
 public class ProxyDistTest {

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/QuarkusPropertiesAutoBuildDistTest.java

@@ -35,13 +35,13 @@ import org.keycloak.it.utils.KeycloakDistribution;
 import io.quarkus.test.junit.main.Launch;
 import io.quarkus.test.junit.main.LaunchResult;
 
-@DistributionTest(reInstall = DistributionTest.ReInstall.NEVER)
+@DistributionTest
 @RawDistOnly(reason = "Containers are immutable")
 @TestMethodOrder(OrderAnnotation.class)
 public class QuarkusPropertiesAutoBuildDistTest {
 
     @Test
-    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false", "--cache=local" })
+    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false" })
     @Order(1)
     void reAugOnFirstRun(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
@@ -50,7 +50,7 @@ public class QuarkusPropertiesAutoBuildDistTest {
 
     @Test
     @BeforeStartDistribution(QuarkusPropertiesAutoBuildDistTest.UpdateConsoleLogLevelToWarn.class)
-    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false", "--cache=local" })
+    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false" })
     @Order(2)
     void testQuarkusRuntimePropDoesNotTriggerReAug(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
@@ -60,7 +60,7 @@ public class QuarkusPropertiesAutoBuildDistTest {
 
     @Test
     @BeforeStartDistribution(UpdateConsoleLogLevelToInfo.class)
-    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false", "--cache=local" })
+    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false" })
     @Order(3)
     void testNoReAugAfterChangingRuntimeProperty(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
@@ -70,7 +70,7 @@ public class QuarkusPropertiesAutoBuildDistTest {
 
     @Test
     @BeforeStartDistribution(AddAdditionalDatasource.class)
-    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false", "--cache=local" })
+    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false" })
     @Order(4)
     void testReAugForAdditionalDatasource(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
@@ -79,7 +79,7 @@ public class QuarkusPropertiesAutoBuildDistTest {
 
     @Test
     @BeforeStartDistribution(ChangeAdditionalDatasourceUsername.class)
-    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false", "--cache=local" })
+    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false" })
     @Order(5)
     void testNoReAugForAdditionalDatasourceRuntimeProperty(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
@@ -88,7 +88,7 @@ public class QuarkusPropertiesAutoBuildDistTest {
 
     @Test
     @BeforeStartDistribution(ChangeAdditionalDatasourceDbKind.class)
-    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false", "--cache=local" })
+    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false" })
     @Order(6)
     void testNoReAugWhenBuildTimePropertiesAreTheSame(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
@@ -97,7 +97,7 @@ public class QuarkusPropertiesAutoBuildDistTest {
 
     @Test
     @BeforeStartDistribution(AddAdditionalDatasource2.class)
-    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false", "--cache=local" })
+    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false" })
     @Order(7)
     void testReAugWhenAnotherDatasourceAdded(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
@@ -106,7 +106,7 @@ public class QuarkusPropertiesAutoBuildDistTest {
 
     @Test
     @BeforeStartDistribution(EnableQuarkusMetrics.class)
-    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false", "--cache=local" })
+    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false" })
     @Order(8)
     void testWrappedBuildPropertyTriggersBuildButGetsIgnoredWhenSetByQuarkus(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/QuarkusPropertiesDistTest.java

@@ -32,6 +32,7 @@ import org.keycloak.it.junit5.extension.BeforeStartDistribution;
 import org.keycloak.it.junit5.extension.CLIResult;
 import org.keycloak.it.junit5.extension.DistributionTest;
 import org.keycloak.it.junit5.extension.KeepServerAlive;
+import org.keycloak.it.junit5.extension.LegacyStore;
 import org.keycloak.it.junit5.extension.RawDistOnly;
 import org.keycloak.it.utils.KeycloakDistribution;
 
@@ -41,13 +42,14 @@ import io.quarkus.test.junit.main.LaunchResult;
 @DistributionTest(reInstall = DistributionTest.ReInstall.NEVER)
 @RawDistOnly(reason = "Containers are immutable")
 @TestMethodOrder(OrderAnnotation.class)
+@LegacyStore
 public class QuarkusPropertiesDistTest {
 
     private static final String QUARKUS_BUILDTIME_HIBERNATE_METRICS_KEY = "quarkus.hibernate-orm.metrics.enabled";
     private static final String QUARKUS_RUNTIME_CONSOLE_LOGLVL_KEY = "quarkus.log.console.level";
 
     @Test
-    @Launch({ "build", "--cache=local" })
+    @Launch({ "build" })
     @Order(1)
     void testBuildWithPropertyFromQuarkusProperties(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
@@ -106,6 +108,7 @@ public class QuarkusPropertiesDistTest {
     @Order(7)
     void testBuildRunTimeMismatchOnQuarkusBuildPropWarning(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
+        cliResult.assertNoBuild();
         cliResult.assertBuildRuntimeMismatchWarning(QUARKUS_BUILDTIME_HIBERNATE_METRICS_KEY);
     }
 

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/StartAutoBuildDistTest.java

@@ -34,13 +34,13 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.keycloak.quarkus.runtime.cli.command.AbstractStartCommand.DEFAULT_WARN_MESSAGE_REPEATED_AUTO_BUILD_OPTION;
 import static org.keycloak.quarkus.runtime.cli.command.AbstractStartCommand.OPTIMIZED_BUILD_OPTION_LONG;
 
-@DistributionTest(reInstall = DistributionTest.ReInstall.NEVER)
+@DistributionTest
 @RawDistOnly(reason = "Containers are immutable")
 @TestMethodOrder(MethodOrderer.OrderAnnotation.class)
 public class StartAutoBuildDistTest {
 
     @Test
-    @Launch({ "start", AbstractStartCommand.AUTO_BUILD_OPTION_LONG, "--http-enabled=true", "--hostname-strict=false", "--cache=local" })
+    @Launch({ "start", AbstractStartCommand.AUTO_BUILD_OPTION_LONG, "--http-enabled=true", "--hostname-strict=false" })
     @Order(1)
     void testStartAutoBuild(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
@@ -56,7 +56,7 @@ public class StartAutoBuildDistTest {
     }
 
     @Test
-    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false", "--cache=local" })
+    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false" })
     @Order(2)
     void testShouldNotReAugIfConfigIsSame(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
@@ -65,7 +65,7 @@ public class StartAutoBuildDistTest {
     }
 
     @Test
-    @Launch({ "start", "--db=dev-mem", "--http-enabled=true", "--hostname-strict=false", "--cache=local" })
+    @Launch({ "start", "--db=dev-mem", "--http-enabled=true", "--hostname-strict=false" })
     @Order(3)
     void testShouldReAugIfConfigChanged(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
@@ -74,7 +74,7 @@ public class StartAutoBuildDistTest {
     }
 
     @Test
-    @Launch({ "start", "--db=dev-mem", "--http-enabled=true", "--hostname-strict=false", "--cache=local" })
+    @Launch({ "start", "--db=dev-mem", "--http-enabled=true", "--hostname-strict=false" })
     @Order(4)
     void testShouldNotReAugIfSameDatabase(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
@@ -91,7 +91,7 @@ public class StartAutoBuildDistTest {
     }
 
     @Test
-    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false", "--cache=local" })
+    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false" })
     @Order(6)
     void testReAugWhenNoOptionAfterBuild(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
@@ -100,7 +100,7 @@ public class StartAutoBuildDistTest {
     }
 
     @Test
-    @Launch({ "start", "--db=postgres", "--http-enabled=true", "--hostname-strict=false", "--cache=local" })
+    @Launch({ "start", "--db=postgres", "--http-enabled=true", "--hostname-strict=false" })
     @Order(7)
     void testShouldReAugWithoutAutoBuildOptionAfterDatabaseChange(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
@@ -109,7 +109,7 @@ public class StartAutoBuildDistTest {
     }
 
     @Test
-    @Launch({ "start", "--db=dev-file", "--http-enabled=true", "--hostname-strict=false", "--cache=local", OPTIMIZED_BUILD_OPTION_LONG})
+    @Launch({ "start", "--db=dev-file", "--http-enabled=true", "--hostname-strict=false", OPTIMIZED_BUILD_OPTION_LONG})
     @Order(8)
     void testShouldReAugAndNeedsAutoBuildOptionBecauseHasNoAutoBuildOption(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/StartCommandDistTest.java

@@ -25,7 +25,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.keycloak.quarkus.runtime.cli.command.AbstractStartCommand.OPTIMIZED_BUILD_OPTION_LONG;
 
 import org.junit.jupiter.api.Test;
-import org.keycloak.it.cli.StartCommandTest;
 import org.keycloak.it.junit5.extension.CLIResult;
 import org.keycloak.it.junit5.extension.DistributionTest;
 
@@ -34,7 +33,35 @@ import io.quarkus.test.junit.main.LaunchResult;
 import org.keycloak.it.utils.KeycloakDistribution;
 
 @DistributionTest
-public class StartCommandDistTest extends StartCommandTest {
+public class StartCommandDistTest {
+
+    @Test
+    @Launch({ "start", "--hostname-strict=false" })
+    void failNoTls(LaunchResult result) {
+        assertTrue(result.getOutput().contains("Key material not provided to setup HTTPS"),
+                () -> "The Output:\n" + result.getOutput() + "doesn't contains the expected string.");
+    }
+
+    @Test
+    @Launch({ "--profile=dev", "start" })
+    void failUsingDevProfile(LaunchResult result) {
+        assertTrue(result.getErrorOutput().contains("ERROR: You can not 'start' the server in development mode. Please re-build the server first, using 'kc.sh build' for the default production mode."),
+                () -> "The Output:\n" + result.getErrorOutput() + "doesn't contains the expected string.");
+    }
+
+    @Test
+    @Launch({ "-v", "start", "--http-enabled=true", "--hostname-strict=false" })
+    void testHttpEnabled(LaunchResult result) {
+        CLIResult cliResult = (CLIResult) result;
+        cliResult.assertStarted();
+    }
+
+    @Test
+    @Launch({ "-v", "start", "--db=dev-mem", OPTIMIZED_BUILD_OPTION_LONG})
+    void failBuildPropertyNotAvailable(LaunchResult result) {
+        CLIResult cliResult = (CLIResult) result;
+        cliResult.assertError("Unknown option: '--db'");
+    }
 
     @Test
     @Launch({ "--profile=dev", "start", "--http-enabled=true", "--hostname-strict=false" })
@@ -52,7 +79,7 @@ public class StartCommandDistTest extends StartCommandTest {
     }
 
     @Test
-    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false", "--cache=local" })
+    @Launch({ "start", "--http-enabled=true", "--hostname-strict=false", "--metrics-enabled=true" })
     void testStartUsingAutoBuild(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
         cliResult.assertMessage("Changes detected in configuration. Updating the server image.");
@@ -61,7 +88,7 @@ public class StartCommandDistTest extends StartCommandTest {
         cliResult.assertMessage(KeycloakDistribution.SCRIPT_CMD + " show-config");
         cliResult.assertMessage("Next time you run the server, just run:");
         cliResult.assertMessage(KeycloakDistribution.SCRIPT_CMD + " start " + OPTIMIZED_BUILD_OPTION_LONG + " --http-enabled=true --hostname-strict=false");
-        assertFalse(cliResult.getOutput().contains("--cache"));
+        assertFalse(cliResult.getOutput().contains("--metrics-enabled"));
         cliResult.assertStarted();
     }
 

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/StartDevCommandDistTest.java

@@ -20,38 +20,43 @@ package org.keycloak.it.cli.dist;
 import io.quarkus.test.junit.main.Launch;
 import io.quarkus.test.junit.main.LaunchResult;
 import org.junit.jupiter.api.MethodOrderer;
-import org.junit.jupiter.api.Order;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestMethodOrder;
-import org.keycloak.it.cli.StartDevCommandTest;
 import org.keycloak.it.junit5.extension.CLIResult;
 import org.keycloak.it.junit5.extension.DistributionTest;
 import org.keycloak.it.junit5.extension.RawDistOnly;
 
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-@DistributionTest(reInstall = DistributionTest.ReInstall.NEVER)
+@DistributionTest
 @RawDistOnly(reason = "Containers are immutable")
 @TestMethodOrder(MethodOrderer.OrderAnnotation.class)
-public class StartDevCommandDistTest extends StartDevCommandTest {
+public class StartDevCommandDistTest {
+
+    @Test
+    @Launch({ "start-dev" })
+    void testDevModeWarning(LaunchResult result) {
+        CLIResult cliResult = (CLIResult) result;
+        cliResult.assertStartedDevMode();
+    }
+
+    @Test
+    @Launch({ "start-dev", "--db=dev-mem" })
+    void testBuildPropertyAvailable(LaunchResult result) {
+        CLIResult cliResult = (CLIResult) result;
+        cliResult.assertStartedDevMode();
+    }
 
     @Test
     @Launch({ "start-dev", "--debug" })
-    @Order(1)
     void testStartDevShouldStartTwoJVMs(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
-        cliResult.assertMessage("Updating the configuration and installing your custom providers, if any. Please wait.");
         cliResult.assertMessageWasShownExactlyNumberOfTimes("Listening for transport dt_socket at address:", 2);
         cliResult.assertStartedDevMode();
     }
 
     @Test
     @Launch({ "build", "--debug" })
-    @Order(2)
     void testBuildMustNotRunTwoJVMs(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
-        cliResult.assertMessage("Updating the configuration and installing your custom providers, if any. Please wait.");
         cliResult.assertMessageWasShownExactlyNumberOfTimes("Listening for transport dt_socket at address:", 1);
         cliResult.assertBuild();
     }

quarkus/tests/integration/src/test/java/org/keycloak/it/storage/database/dist/DatabaseOptionsDistTest.java

@@ -20,6 +20,7 @@ package org.keycloak.it.storage.database.dist;
 import org.junit.jupiter.api.Test;
 import org.keycloak.it.junit5.extension.CLIResult;
 import org.keycloak.it.junit5.extension.DistributionTest;
+import org.keycloak.it.junit5.extension.LegacyStore;
 import org.keycloak.it.junit5.extension.WithDatabase;
 import org.keycloak.it.utils.KeycloakDistribution;
 
@@ -28,6 +29,7 @@ import io.quarkus.test.junit.main.LaunchResult;
 
 @DistributionTest
 @WithDatabase(alias = "postgres")
+@LegacyStore
 public class DatabaseOptionsDistTest {
 
     @Test

quarkus/tests/integration/src/test/java/org/keycloak/it/storage/database/dist/MariaDBDistTest.java

@@ -19,11 +19,13 @@ package org.keycloak.it.storage.database.dist;
 
 import org.keycloak.it.junit5.extension.CLITest;
 import org.keycloak.it.junit5.extension.DistributionTest;
+import org.keycloak.it.junit5.extension.LegacyStore;
 import org.keycloak.it.junit5.extension.WithDatabase;
 import org.keycloak.it.storage.database.MariaDBTest;
 
 @DistributionTest(removeBuildOptionsAfterBuild = true)
 @WithDatabase(alias = "mariadb")
+@LegacyStore
 public class MariaDBDistTest extends MariaDBTest {
 
 }

quarkus/tests/integration/src/test/java/org/keycloak/it/storage/database/dist/MySQLDistTest.java

@@ -1,10 +1,12 @@
 package org.keycloak.it.storage.database.dist;
 
 import org.keycloak.it.junit5.extension.DistributionTest;
+import org.keycloak.it.junit5.extension.LegacyStore;
 import org.keycloak.it.junit5.extension.WithDatabase;
 import org.keycloak.it.storage.database.MySQLTest;
 
 @DistributionTest(removeBuildOptionsAfterBuild = true)
 @WithDatabase(alias = "mysql")
+@LegacyStore
 public class MySQLDistTest extends MySQLTest {
 }

quarkus/tests/integration/src/test/java/org/keycloak/it/storage/database/dist/PostgreSQLDistTest.java

@@ -23,6 +23,7 @@ import static org.hamcrest.MatcherAssert.assertThat;
 import org.junit.jupiter.api.Test;
 import org.keycloak.it.junit5.extension.CLIResult;
 import org.keycloak.it.junit5.extension.DistributionTest;
+import org.keycloak.it.junit5.extension.LegacyStore;
 import org.keycloak.it.junit5.extension.WithDatabase;
 import org.keycloak.it.storage.database.PostgreSQLTest;
 
@@ -31,6 +32,7 @@ import io.quarkus.test.junit.main.LaunchResult;
 
 @DistributionTest(removeBuildOptionsAfterBuild = true)
 @WithDatabase(alias = "postgres")
+@LegacyStore
 public class PostgreSQLDistTest extends PostgreSQLTest {
 
     @Test

quarkus/tests/integration/src/test/java/org/keycloak/it/storage/map/ChmStorageDistTest.java

@@ -28,7 +28,7 @@ import io.quarkus.test.junit.main.Launch;
 import io.quarkus.test.junit.main.LaunchResult;
 
 @RawDistOnly(reason = "Need to check dist path")
-@DistributionTest(reInstall = DistributionTest.ReInstall.BEFORE_TEST)
+@DistributionTest
 public class ChmStorageDistTest {
 
     @Test