Keep consistency when importing realms at startup when they are exported via the export command

Closes #16281

Co-authored-by: Martin Bartoš <mabartos@redhat.com>

docs/guides/src/main/server/importExport.adoc

@@ -79,12 +79,16 @@ You are also able to import realms when the server is starting by using the `--i
 
 <@kc.start parameters="--import-realm"/>
 
-When you set the `--import-realm` option, the server is going to try to import any realm configuration file from the `data/import` directory. Each file in this directory should
+When you set the `--import-realm` option, the server is going to try to import any realm configuration file from the `data/import` directory. Only regular files using the `.json` extension are read from this directory, sub-directories are ignored.
-contain a single realm configuration. Only regular files using the `.json` extension are read from this directory, sub-directories are ignored.
 
 NOTE: For the https://quay.io/keycloak/keycloak[published containers], the import directory is `/opt/keycloak/data/import`
 
-If a realm already exists in the server, the import operation is skipped.
+If a realm already exists in the server, the import operation is skipped. The main reason behind this behavior is to avoid re-creating
+realms and potentially loose state between server restarts.
+
+To re-create realms you should explicitly run the `import` command prior to starting the server.
+
+Importing the `master` realm is not supported because as it is a very sensitive operation.
 
 === Using Environment Variables within the Realm Configuration Files
 

model/legacy-services/src/main/java/org/keycloak/exportimport/AbstractFileBasedImportProvider.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2022 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.exportimport;
+
+import static org.keycloak.common.util.StringPropertyReplacer.replaceProperties;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.util.Optional;
+import org.keycloak.common.util.StringPropertyReplacer;
+
+public abstract class AbstractFileBasedImportProvider implements ImportProvider {
+
+    private static final StringPropertyReplacer.PropertyResolver ENV_VAR_PROPERTY_RESOLVER = new StringPropertyReplacer.PropertyResolver() {
+        @Override
+        public String resolve(String property) {
+            return Optional.ofNullable(property).map(System::getenv).orElse(null);
+        }
+    };
+
+    protected InputStream parseFile(File importFile) throws IOException {
+        if (ExportImportConfig.isReplacePlaceholders()) {
+            String raw = new String(Files.readAllBytes(importFile.toPath()), "UTF-8");
+            String parsed = replaceProperties(raw, ENV_VAR_PROPERTY_RESOLVER);
+            return new ByteArrayInputStream(parsed.getBytes());
+        }
+
+        return new FileInputStream(importFile);
+    }
+
+}

model/legacy-services/src/main/java/org/keycloak/exportimport/dir/DirImportProvider.java

@@ -19,7 +19,7 @@ package org.keycloak.exportimport.dir;
 
 import org.jboss.logging.Logger;
 import org.keycloak.Config;
-import org.keycloak.exportimport.ImportProvider;
+import org.keycloak.exportimport.AbstractFileBasedImportProvider;
 import org.keycloak.exportimport.Strategy;
 import org.keycloak.exportimport.util.ExportImportSessionTask;
 import org.keycloak.exportimport.util.ImportUtils;
@@ -31,9 +31,9 @@ import org.keycloak.representations.idm.RealmRepresentation;
 import org.keycloak.util.JsonSerialization;
 
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.FilenameFilter;
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -42,7 +42,7 @@ import org.keycloak.services.managers.RealmManager;
 /**
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
-public class DirImportProvider implements ImportProvider {
+public class DirImportProvider extends AbstractFileBasedImportProvider {
 
     private static final Logger logger = Logger.getLogger(DirImportProvider.class);
 
@@ -127,7 +127,7 @@ public class DirImportProvider implements ImportProvider {
         });
 
         // Import realm first
-        FileInputStream is = new FileInputStream(realmFile);
+        InputStream is = parseFile(realmFile);
         final RealmRepresentation realmRep = JsonSerialization.readValue(is, RealmRepresentation.class);
         final AtomicBoolean realmImported = new AtomicBoolean();
 
@@ -144,7 +144,7 @@ public class DirImportProvider implements ImportProvider {
         if (realmImported.get()) {
             // Import users
             for (final File userFile : userFiles) {
-                final FileInputStream fis = new FileInputStream(userFile);
+                final InputStream fis = parseFile(userFile);
                 KeycloakModelUtils.runJobInTransaction(factory, new ExportImportSessionTask() {
                     @Override
                     protected void runExportImportTask(KeycloakSession session) throws IOException {
@@ -154,7 +154,7 @@ public class DirImportProvider implements ImportProvider {
                 });
             }
             for (final File userFile : federatedUserFiles) {
-                final FileInputStream fis = new FileInputStream(userFile);
+                final InputStream fis = parseFile(userFile);
                 KeycloakModelUtils.runJobInTransaction(factory, new ExportImportSessionTask() {
                     @Override
                     protected void runExportImportTask(KeycloakSession session) throws IOException {
@@ -165,16 +165,18 @@ public class DirImportProvider implements ImportProvider {
             }
         }
 
-        // Import authorization and initialize service accounts last, as they require users already in DB
+        if (realmImported.get()) {
-        KeycloakModelUtils.runJobInTransaction(factory, new ExportImportSessionTask() {
+            // Import authorization and initialize service accounts last, as they require users already in DB
+            KeycloakModelUtils.runJobInTransaction(factory, new ExportImportSessionTask() {
 
-            @Override
+                @Override
-            public void runExportImportTask(KeycloakSession session) throws IOException {
+                public void runExportImportTask(KeycloakSession session) throws IOException {
-                RealmManager realmManager = new RealmManager(session);
+                    RealmManager realmManager = new RealmManager(session);
-                realmManager.setupClientServiceAccountsAndAuthorizationOnImport(realmRep, false);
+                    realmManager.setupClientServiceAccountsAndAuthorizationOnImport(realmRep, false);
-            }
+                }
 
-        });
+            });
+        }
     }
 
     @Override

model/legacy-services/src/main/java/org/keycloak/exportimport/singlefile/SingleFileImportProvider.java

@@ -19,7 +19,7 @@ package org.keycloak.exportimport.singlefile;
 
 import org.jboss.logging.Logger;
 import org.keycloak.Config;
-import org.keycloak.exportimport.ImportProvider;
+import org.keycloak.exportimport.AbstractFileBasedImportProvider;
 import org.keycloak.exportimport.Strategy;
 import org.keycloak.exportimport.util.ExportImportSessionTask;
 import org.keycloak.exportimport.util.ImportUtils;
@@ -30,14 +30,14 @@ import org.keycloak.representations.idm.RealmRepresentation;
 import org.keycloak.util.JsonSerialization;
 
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.Map;
 
 /**
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
-public class SingleFileImportProvider implements ImportProvider {
+public class SingleFileImportProvider extends AbstractFileBasedImportProvider {
 
     private static final Logger logger = Logger.getLogger(SingleFileImportProvider.class);
 
@@ -73,7 +73,7 @@ public class SingleFileImportProvider implements ImportProvider {
 
     protected void checkRealmReps() throws IOException {
         if (realmReps == null) {
-            FileInputStream is = new FileInputStream(file);
+            InputStream is = parseFile(file);
             realmReps = ImportUtils.getRealmsFromStream(JsonSerialization.mapper, is);
         }
     }

quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/cli/command/ImportRealmMixin.java

@@ -40,13 +40,7 @@ public final class ImportRealmMixin {
         File importDir = Environment.getHomePath().resolve("data").resolve("import").toFile();
 
         if (importDir.exists()) {
-            StringBuilder filesToImport = new StringBuilder();
+            System.setProperty("keycloak.import", importDir.getAbsolutePath());
-
-            for (File realmFile : importDir.listFiles()) {
-                filesToImport.append(realmFile.getAbsolutePath()).append(",");
-            }
-
-            System.setProperty("keycloak.import", filesToImport.toString());
         }
     }
 }

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/ImportAtStartupDistTest.java

@@ -17,6 +17,7 @@
 
 package org.keycloak.it.cli.dist;
 
+import java.io.IOException;
 import java.nio.file.Path;
 import java.util.function.Consumer;
 import org.junit.jupiter.api.Test;
@@ -29,6 +30,7 @@ import org.keycloak.it.junit5.extension.RawDistOnly;
 import org.keycloak.it.utils.KeycloakDistribution;
 import org.keycloak.it.utils.RawKeycloakDistribution;
 
+import io.quarkus.deployment.util.FileUtil;
 import io.quarkus.test.junit.main.Launch;
 import io.quarkus.test.junit.main.LaunchResult;
 
@@ -41,21 +43,21 @@ public class ImportAtStartupDistTest {
     @Launch({"start-dev", "--import-realm"})
     void testImport(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
-        cliResult.assertMessage("Imported realm quickstart-realm from file");
+        cliResult.assertMessage("Realm 'quickstart-realm' imported");
     }
 
     @Test
     @BeforeStartDistribution(CreateRealmConfigurationFileAndDir.class)
-    @Launch({"start-dev", "--import-realm", "--log-level=org.keycloak.services.resources.KeycloakApplication:debug"})
+    @Launch({"start-dev", "--import-realm", "--log-level=org.keycloak.exportimport.ExportImportManager:debug"})
     void testImportAndIgnoreDirectory(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
-        cliResult.assertMessage("Imported realm quickstart-realm from file");
+        cliResult.assertMessage("Realm 'quickstart-realm' imported");
         cliResult.assertMessage("Ignoring import file because it is not a valid file");
     }
 
     @Test
     @BeforeStartDistribution(CreateRealmConfigurationFileWithUnsupportedExtension.class)
-    @Launch({"start-dev", "--import-realm", "--log-level=org.keycloak.services.resources.KeycloakApplication:debug"})
+    @Launch({"start-dev", "--import-realm", "--log-level=org.keycloak.exportimport.ExportImportManager:debug"})
     void testIgnoreFileWithUnsupportedExtension(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
         cliResult.assertMessage("Ignoring import file because it is not a valid file");
@@ -70,6 +72,49 @@ public class ImportAtStartupDistTest {
         cliResult.assertError("option '--import-realm' should be specified without 'some-file' parameter");
     }
 
+    @Test
+    @BeforeStartDistribution(CreateRealmConfigurationFile.class)
+    void testImportFromFileCreatedByExportAllRealms(KeycloakDistribution dist) throws IOException {
+        dist.run("start-dev", "--import-realm");
+        dist.run("export", "--file=../data/import/realm.json");
+
+        RawKeycloakDistribution rawDist = dist.unwrap(RawKeycloakDistribution.class);
+        FileUtil.deleteDirectory(rawDist.getDistPath().resolve("data").resolve("chm").toAbsolutePath());
+
+        CLIResult result = dist.run("start-dev", "--import-realm");
+        result.assertMessage("Realm 'quickstart-realm' imported");
+        result.assertMessage("Realm 'master' already exists. Import skipped");
+    }
+
+    @Test
+    @BeforeStartDistribution(CreateRealmConfigurationFile.class)
+    void testImportFromFileCreatedByExportSingleRealm(KeycloakDistribution dist) throws IOException {
+        dist.run("start-dev", "--import-realm");
+        dist.run("export", "--realm=quickstart-realm", "--file=../data/import/realm.json");
+
+        RawKeycloakDistribution rawDist = dist.unwrap(RawKeycloakDistribution.class);
+        FileUtil.deleteDirectory(rawDist.getDistPath().resolve("data").resolve("chm").toAbsolutePath());
+
+        CLIResult result = dist.run("start-dev", "--import-realm");
+        result.assertMessage("Realm 'quickstart-realm' imported");
+        result.assertNoMessage("Not importing realm master from file");
+    }
+
+    @Test
+    @BeforeStartDistribution(CreateRealmConfigurationFile.class)
+    void testImportFromDirCreatedByExport(KeycloakDistribution dist) throws IOException {
+        dist.run("start-dev", "--import-realm");
+        RawKeycloakDistribution rawDist = dist.unwrap(RawKeycloakDistribution.class);
+        FileUtil.deleteDirectory(rawDist.getDistPath().resolve("data").resolve("import").toAbsolutePath());
+        dist.run("export", "--dir=../data/import");
+
+        FileUtil.deleteDirectory(rawDist.getDistPath().resolve("data").resolve("chm").toAbsolutePath());
+
+        CLIResult result = dist.run("start-dev", "--import-realm");
+        result.assertMessage("Realm 'quickstart-realm' imported");
+        result.assertNoMessage("Not importing realm master from file");
+    }
+
     public static class CreateRealmConfigurationFile implements Consumer<KeycloakDistribution> {
 
         @Override

services/src/main/java/org/keycloak/exportimport/ExportImportConfig.java

@@ -39,6 +39,9 @@ public class ExportImportConfig {
     // used for "singleFile" provider
     public static final String FILE = PREFIX + "file";
 
+    // used for replacing placeholders
+    public static final String REPLACE_PLACEHOLDERS = PREFIX + "replace-placeholders";
+
     // How to export users when realm export is requested for "dir" provider
     public static final String USERS_EXPORT_STRATEGY = PREFIX + "usersExportStrategy";
     public static final UsersExportStrategy DEFAULT_USERS_EXPORT_STRATEGY = UsersExportStrategy.DIFFERENT_FILES;
@@ -117,4 +120,12 @@ public class ExportImportConfig {
         String strategy = System.getProperty(STRATEGY, DEFAULT_STRATEGY.toString());
         return Enum.valueOf(Strategy.class, strategy);
     }
+
+    public static boolean isReplacePlaceholders() {
+        return Boolean.getBoolean(REPLACE_PLACEHOLDERS);
+    }
+
+    public static void setReplacePlaceholders(boolean replacePlaceholders) {
+        System.setProperty(REPLACE_PLACEHOLDERS, String.valueOf(replacePlaceholders));
+    }
 }

services/src/main/java/org/keycloak/exportimport/ExportImportManager.java

@@ -21,9 +21,20 @@ package org.keycloak.exportimport;
 import org.jboss.logging.Logger;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
+import org.keycloak.models.KeycloakSessionTask;
+import org.keycloak.models.utils.KeycloakModelUtils;
+import org.keycloak.provider.ProviderFactory;
 import org.keycloak.services.ServicesLogger;
 
+import java.io.File;
 import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
@@ -33,6 +44,7 @@ public class ExportImportManager {
     private static final Logger logger = Logger.getLogger(ExportImportManager.class);
 
     private KeycloakSessionFactory sessionFactory;
+    private KeycloakSession session;
 
     private final String realmName;
 
@@ -41,6 +53,7 @@ public class ExportImportManager {
 
     public ExportImportManager(KeycloakSession session) {
         this.sessionFactory = session.getKeycloakSessionFactory();
+        this.session = session;
 
         realmName = ExportImportConfig.getRealmName();
 
@@ -95,6 +108,57 @@ public class ExportImportManager {
         }
     }
 
+    public void runImportAtStartup(String dir, Strategy strategy) throws IOException {
+        ExportImportConfig.setReplacePlaceholders(true);
+        ExportImportConfig.setAction("import");
+
+        Stream<ProviderFactory> factories = sessionFactory.getProviderFactoriesStream(ImportProvider.class);
+
+        for (ProviderFactory factory : factories.collect(Collectors.toList())) {
+            String providerId = factory.getId();
+
+            if ("dir".equals(providerId)) {
+                ExportImportConfig.setDir(dir);
+                ImportProvider importProvider = session.getProvider(ImportProvider.class, providerId);
+                importProvider.importModel(sessionFactory, strategy);
+            } else if ("singleFile".equals(providerId)) {
+                Set<String> filesToImport = new HashSet<>();
+
+                for (File file : Paths.get(dir).toFile().listFiles()) {
+                    Path filePath = file.toPath();
+
+                    if (!(Files.exists(filePath) && Files.isRegularFile(filePath) && filePath.toString().endsWith(".json"))) {
+                        logger.debugf("Ignoring import file because it is not a valid file: %s", file);
+                        continue;
+                    }
+
+                    String fileName = file.getName();
+
+                    if (fileName.contains("-realm.json") || fileName.contains("-users-")) {
+                        continue;
+                    }
+
+                    filesToImport.add(file.getAbsolutePath());
+                }
+
+                for (String file : filesToImport) {
+                    ExportImportConfig.setFile(file);
+                    KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
+                        @Override
+                        public void run(KeycloakSession session) {
+                            ImportProvider importProvider = session.getProvider(ImportProvider.class, providerId);
+                            try {
+                                importProvider.importModel(sessionFactory, strategy);
+                            } catch (IOException cause) {
+                                throw new RuntimeException(cause);
+                            }
+                        }
+                    });
+                }
+            }
+        }
+    }
+
     public void runExport() {
         try {
             if (realmName == null) {

services/src/main/java/org/keycloak/services/resources/KeycloakApplication.java

@@ -22,9 +22,9 @@ import org.keycloak.Config;
 import org.keycloak.common.Profile;
 import org.keycloak.common.crypto.CryptoIntegration;
 import org.keycloak.common.util.Resteasy;
-import org.keycloak.common.util.StringPropertyReplacer;
 import org.keycloak.config.ConfigProviderFactory;
 import org.keycloak.exportimport.ExportImportManager;
+import org.keycloak.exportimport.Strategy;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
 import org.keycloak.models.KeycloakSessionTask;
@@ -60,16 +60,11 @@ import javax.ws.rs.core.Application;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.HashSet;
 import java.util.List;
 import java.util.NoSuchElementException;
-import java.util.Optional;
 import java.util.ServiceLoader;
 import java.util.Set;
-import java.util.StringTokenizer;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
@@ -220,7 +215,7 @@ public class KeycloakApplication extends Application {
         if (exportImportManager[0].isRunImport()) {
             exportImportManager[0].runImport();
         } else {
-            importRealms();
+            importRealms(exportImportManager[0]);
         }
 
         importAddUser();
@@ -262,32 +257,13 @@ public class KeycloakApplication extends Application {
         return singletons;
     }
 
-    public void importRealms() {
+    public void importRealms(ExportImportManager exportImportManager) {
-        String files = System.getProperty("keycloak.import");
+        String dir = System.getProperty("keycloak.import");
-        if (files != null) {
+        if (dir != null) {
-            StringTokenizer tokenizer = new StringTokenizer(files, ",");
+            try {
-            while (tokenizer.hasMoreTokens()) {
+                exportImportManager.runImportAtStartup(dir, Strategy.IGNORE_EXISTING);
-                String file = tokenizer.nextToken().trim();
+            } catch (IOException cause) {
-                RealmRepresentation rep;
+                throw new RuntimeException("Failed to import realms", cause);
-                try {
-                    Path filePath = Paths.get(file);
-
-                    if (!(Files.exists(filePath) && Files.isRegularFile(filePath) && filePath.toString().endsWith(".json"))) {
-                        logger.debugf("Ignoring import file because it is not a valid file: %s", file);
-                        continue;
-                    }
-
-                    rep = JsonSerialization.readValue(StringPropertyReplacer.replaceProperties(
-                            new String(Files.readAllBytes(filePath), "UTF-8"), new StringPropertyReplacer.PropertyResolver() {
-                                @Override
-                                public String resolve(String property) {
-                                    return Optional.ofNullable(System.getenv(property)).orElse(null);
-                                }
-                            }), RealmRepresentation.class);
-                } catch (Exception cause) {
-                    throw new RuntimeException("Failed to parse realm configuration file: " + file, cause);
-                }
-                importRealm(rep, "file " + file);
             }
         }
     }