fix: allows for the detection of a master realm with --import-realms (#32914)

also moving initial bootstrapping after import

closes: #32689

Signed-off-by: Steven Hawkins <shawkins@redhat.com>
Co-authored-by: Martin Bartoš <mabartos@redhat.com>

docs/documentation/upgrading/topics/changes/changes-26_0_0.adoc

@@ -300,6 +300,10 @@ The new event types are supported by the Email Event Listener.
 
 The following event types are now deprecated and will be removed in a future version: `UPDATE_PASSWORD`, `UPDATE_PASSWORD_ERROR`, `UPDATE_TOTP`, `UPDATE_TOTP_ERROR`, `REMOVE_TOTP`, `REMOVE_TOTP_ERROR`
 
+= `--import-realm` option can import the master realm
+
+When running a `start` or `start-dev` command with the `--import-realm` option before the master realm exists, it will be imported if it exists in the import material. The previous behavior was that the master realm was created first, then its import skipped. 
+
 = BouncyCastle FIPS updated
 
 Our FIPS 140-2 integration is now tested and supported with version 2 of BouncyCastle FIPS libraries. This version is certified with Java 21. If you use FIPS 140-2 integration, it is recommended to

model/storage-services/src/main/java/org/keycloak/exportimport/util/ExportImportSessionTask.java

@@ -37,6 +37,11 @@ public abstract class ExportImportSessionTask implements KeycloakSessionTask {
             throw new RuntimeException("Error during export/import: " + ioe.getMessage(), ioe);
         }
     }
+    
+    @Override
+    public boolean useExistingSession() {
+        return true;
+    }
 
     protected abstract void runExportImportTask(KeycloakSession session) throws IOException;
 }

quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/cli/command/ImportRealmMixin.java

@@ -20,7 +20,6 @@ package org.keycloak.quarkus.runtime.cli.command;
 import static org.keycloak.quarkus.runtime.cli.Picocli.NO_PARAM_LABEL;
 
 import java.io.File;
-import java.util.Optional;
 import org.keycloak.quarkus.runtime.Environment;
 
 import picocli.CommandLine;

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/ImportAtStartupDistTest.java

@@ -83,7 +83,8 @@ public class ImportAtStartupDistTest {
 
         CLIResult result = dist.run("start-dev", "--import-realm");
         result.assertMessage("Realm 'quickstart-realm' imported");
-        result.assertMessage("Realm 'master' already exists. Import skipped");
+        result.assertMessage("Realm 'master' imported");
+        result.assertNoMessage("Realm 'master' already exists. Import skipped");
     }
 
     @Test

server-spi-private/src/main/java/org/keycloak/models/utils/KeycloakModelUtils.java

@@ -262,7 +262,7 @@ public final class KeycloakModelUtils {
         runJobInTransactionWithResult(factory, null, session -> {
             task.run(session);
             return null;
-        });
+        }, task.useExistingSession());
     }
 
     /**
@@ -275,7 +275,7 @@ public final class KeycloakModelUtils {
         runJobInTransactionWithResult(factory, context, session -> {
             task.run(session);
             return null;
-        });
+        }, task.useExistingSession());
     }
 
     /**
@@ -365,7 +365,7 @@ public final class KeycloakModelUtils {
      * @return The return value from the callable
      */
     public static <V> V runJobInTransactionWithResult(KeycloakSessionFactory factory, final KeycloakSessionTaskWithResult<V> callable) {
-        return runJobInTransactionWithResult(factory, null, callable);
+        return runJobInTransactionWithResult(factory, null, callable, false);
     }
 
     /**
@@ -374,13 +374,19 @@ public final class KeycloakModelUtils {
      * @param factory The session factory
      * @param context The context from the previous session to use
      * @param callable The callable to execute
+     * @param useExistingSession if the existing session should be used
      * @return The return value from the callable
      */
-    public static <V> V runJobInTransactionWithResult(KeycloakSessionFactory factory, KeycloakContext context, final KeycloakSessionTaskWithResult<V> callable) {
+    public static <V> V runJobInTransactionWithResult(KeycloakSessionFactory factory, KeycloakContext context, final KeycloakSessionTaskWithResult<V> callable, boolean useExistingSession) {
         V result;
+        KeycloakSession existing = KeycloakSessionUtil.getKeycloakSession();
+        if (useExistingSession && existing != null && existing.getTransactionManager().isActive()) {
+            return callable.run(existing);
+        }
+        
         try (KeycloakSession session = factory.create()) {
             session.getTransactionManager().begin();
-            KeycloakSession old = KeycloakSessionUtil.setKeycloakSession(session);
+            KeycloakSessionUtil.setKeycloakSession(session);
             try {
                 cloneContextRealmClientToSession(context, session);
                 result = callable.run(session);
@@ -388,7 +394,7 @@ public final class KeycloakModelUtils {
                 session.getTransactionManager().setRollbackOnly();
                 throw t;
             } finally {
-                KeycloakSessionUtil.setKeycloakSession(old);
+                KeycloakSessionUtil.setKeycloakSession(existing);
             }
         }
         return result;

server-spi/src/main/java/org/keycloak/models/KeycloakSessionTask.java

@@ -25,5 +25,9 @@ package org.keycloak.models;
 public interface KeycloakSessionTask {
 
     void run(KeycloakSession session);
+    
+    default boolean useExistingSession() {
+        return false;
+    }
 
 }

services/src/main/java/org/keycloak/exportimport/ExportImportManager.java

@@ -21,8 +21,6 @@ import org.jboss.logging.Logger;
 import org.keycloak.Config;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
-import org.keycloak.models.KeycloakSessionTask;
-import org.keycloak.models.utils.KeycloakModelUtils;
 import org.keycloak.provider.ProviderFactory;
 
 import java.io.File;
@@ -33,7 +31,7 @@ import java.nio.file.Paths;
 import java.util.HashSet;
 import java.util.Objects;
 import java.util.Set;
-import java.util.stream.Collectors;
+import java.util.function.Supplier;
 import java.util.stream.Stream;
 
 import static org.keycloak.exportimport.ExportImportConfig.PROVIDER;
@@ -92,6 +90,20 @@ public class ExportImportManager {
             throw new RuntimeException(e);
         }
     }
+    
+    public boolean isImportMasterIncludedAtStartup(String dir) {
+        if (dir == null) {
+            throw new IllegalStateException("Import not enabled");
+        }
+        
+        return getStartupImportProviders(dir).map(Supplier::get).anyMatch(provider -> {
+            try {
+                return provider.isMasterRealmExported();
+            } catch (IOException e) {
+                throw new RuntimeException("Failed to run import", e);
+            }
+        });
+    }
 
     public boolean isRunExport() {
         return exportProvider != null;
@@ -104,21 +116,38 @@ public class ExportImportManager {
             throw new RuntimeException("Failed to run import", e);
         }
     }
-
+    
     public void runImportAtStartup(String dir) throws IOException {
+        System.setProperty(ExportImportConfig.STRATEGY, Strategy.IGNORE_EXISTING.toString());
         ExportImportConfig.setReplacePlaceholders(true);
-        ExportImportConfig.setAction("import");
+        // enables logging of what is imported
+        ExportImportConfig.setAction(ExportImportConfig.ACTION_IMPORT);
+        
+        // TODO: ideally the static setting above should be unset after this is run 
+        
+        getStartupImportProviders(dir).map(Supplier::get).forEach(ip -> {
+            try {
+                ip.importModel();
+            } catch (IOException e) {
+                throw new RuntimeException("Failed to run import", e);
+            }
+        });
+    }
 
+    private Stream<Supplier<ImportProvider>> getStartupImportProviders(String dir) {
         Stream<ProviderFactory> factories = sessionFactory.getProviderFactoriesStream(ImportProvider.class);
 
-        for (ProviderFactory factory : factories.collect(Collectors.toList())) {
+        return factories.flatMap(factory -> {
             String providerId = factory.getId();
 
             if ("dir".equals(providerId)) {
-                ExportImportConfig.setDir(dir);
+                Supplier<ImportProvider> func = () -> {
-                ImportProvider importProvider = session.getProvider(ImportProvider.class, providerId);
+                    ExportImportConfig.setDir(dir);
-                importProvider.importModel();
+                    return session.getProvider(ImportProvider.class, providerId);
-            } else if ("singleFile".equals(providerId)) {
+                };
+                return Stream.of(func);
+            }
+            if ("singleFile".equals(providerId)) {
                 Set<String> filesToImport = new HashSet<>();
 
                 File[] files = Paths.get(dir).toFile().listFiles();
@@ -139,23 +168,14 @@ public class ExportImportManager {
 
                     filesToImport.add(file.getAbsolutePath());
                 }
-
+                
-                for (String file : filesToImport) {
+                return filesToImport.stream().map(file -> () -> {
                     ExportImportConfig.setFile(file);
-                    KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
+                    return session.getProvider(ImportProvider.class, providerId);
-                        @Override
+                });
-                        public void run(KeycloakSession session) {
-                            ImportProvider importProvider = session.getProvider(ImportProvider.class, providerId);
-                            try {
-                                importProvider.importModel();
-                            } catch (IOException cause) {
-                                throw new RuntimeException(cause);
-                            }
-                        }
-                    });
-                }
             }
-        }
+            return Stream.empty();
+        });
     }
 
     public void runExport() {

services/src/main/java/org/keycloak/services/resources/KeycloakApplication.java

@@ -20,9 +20,7 @@ import org.jboss.logging.Logger;
 import org.keycloak.Config;
 import org.keycloak.common.crypto.CryptoIntegration;
 import org.keycloak.config.ConfigProviderFactory;
-import org.keycloak.exportimport.ExportImportConfig;
 import org.keycloak.exportimport.ExportImportManager;
-import org.keycloak.exportimport.Strategy;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
 import org.keycloak.models.KeycloakSessionTask;
@@ -50,6 +48,7 @@ import java.io.FileInputStream;
 import java.io.IOException;
 import java.util.List;
 import java.util.NoSuchElementException;
+import java.util.Optional;
 import java.util.ServiceLoader;
 
 import com.fasterxml.jackson.core.type.TypeReference;
@@ -118,10 +117,15 @@ public abstract class KeycloakApplication extends Application {
         if (sessionFactory != null)
             sessionFactory.close();
     }
+    
+    private static class BootstrapState {
+        ExportImportManager exportImportManager;
+        boolean newInstall;
+    }
 
     // Bootstrap master realm, import realms and create admin user.
     protected ExportImportManager bootstrap() {
-        ExportImportManager[] exportImportManager = new ExportImportManager[1];
+        BootstrapState bootstrapState = new BootstrapState();
 
         logger.debug("bootstrap");
         KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
@@ -145,29 +149,38 @@ public abstract class KeycloakApplication extends Application {
                 // TODO up here ^^
 
                 ApplianceBootstrap applianceBootstrap = new ApplianceBootstrap(session);
-                exportImportManager[0] = new ExportImportManager(session);
+                var exportImportManager = bootstrapState.exportImportManager = new ExportImportManager(session);
-
+                bootstrapState.newInstall = applianceBootstrap.isNewInstall();
-                boolean createMasterRealm = applianceBootstrap.isNewInstall();
+                if (bootstrapState.newInstall) {
-                if (exportImportManager[0].isRunImport() && exportImportManager[0].isImportMasterIncluded()) {
+                    // check if this is an import command that is importing the master realm
-                    createMasterRealm = false;
+                    boolean importingMaster = exportImportManager.isRunImport() && exportImportManager.isImportMasterIncluded();
-                }
+                    // check if this is a start command that is importing the master realm
-
+                    importingMaster |= getImportDirectory().filter(exportImportManager::isImportMasterIncludedAtStartup).isPresent();
-                if (createMasterRealm) {
+                    if (!importingMaster) {
-                    applianceBootstrap.createMasterRealm();
+                        applianceBootstrap.createMasterRealm();
+                    }
+                    // these are also running in the initial bootstrap transaction - if there is a problem, the server won't be initialized at all
+                    runImports(exportImportManager);
                     createTemporaryAdmin(session);
-                }
+                } 
             }
         });
 
-        if (exportImportManager[0].isRunImport()) {
+        if (!bootstrapState.newInstall) {
-            exportImportManager[0].runImport();
+            runImports(bootstrapState.exportImportManager);
-        } else {
-            importRealms(exportImportManager[0]);
         }
-
+        
         importAddUser();
 
-        return exportImportManager[0];
+        return bootstrapState.exportImportManager;
+    }
+
+    private void runImports(ExportImportManager exportImportManager) {
+        if (exportImportManager.isRunImport()) {
+            exportImportManager.runImport();
+        } else {
+            importRealms(exportImportManager);
+        }
     }
 
     protected abstract void createTemporaryAdmin(KeycloakSession session);
@@ -193,15 +206,17 @@ public abstract class KeycloakApplication extends Application {
     }
 
     public void importRealms(ExportImportManager exportImportManager) {
-        String dir = System.getProperty("keycloak.import");
+        getImportDirectory().ifPresent(dir -> {
-        if (dir != null) {
             try {
-                System.setProperty(ExportImportConfig.STRATEGY, Strategy.IGNORE_EXISTING.toString());
                 exportImportManager.runImportAtStartup(dir);
             } catch (IOException cause) {
                 throw new RuntimeException("Failed to import realms", cause);
             }
-        }
+        });
+    }
+
+    private Optional<String> getImportDirectory() {
+        return Optional.ofNullable(System.getProperty("keycloak.import"));
     }
 
     public void importRealm(RealmRepresentation rep, String from) {

services/src/main/java/org/keycloak/services/resources/admin/RealmAdminResource.java

@@ -1117,7 +1117,7 @@ public class RealmAdminResource {
                         AdminEventBuilder adminEventClone = adminEvent.clone(kcSession);
                         // calling a static method to avoid using the wrong instances
                         return getPartialImportResults(requestBody, kcSession, realmClone, adminEventClone);
-                    })
+                    }, false)
             ).build();
         } catch (ModelDuplicateException e) {
             throw ErrorResponse.exists(e.getLocalizedMessage());

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/model/ClientModelTest.java

@@ -299,7 +299,7 @@ public class ClientModelTest extends AbstractKeycloakTest {
 
             client = realm.addClient(id, "application2");
             return client.getId();
-        });
+        }, false);
 
         KeycloakModelUtils.runJobInTransaction(session.getKeycloakSessionFactory(), session.getContext(), (KeycloakSession sessionAppWithId2) -> {
             currentSession = sessionAppWithId2;