From 522bf1c0b0490fd9a6a8bf18a6b06aae0fa989c5 Mon Sep 17 00:00:00 2001 From: Pedro Igor Date: Thu, 5 Jan 2023 14:31:21 -0300 Subject: [PATCH] Keep consistency when importing realms at startup when they are exported via the export command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #16281 Co-authored-by: Martin Bartoš --- docs/guides/src/main/server/importExport.adoc | 10 ++- .../AbstractFileBasedImportProvider.java | 50 +++++++++++++++ .../exportimport/dir/DirImportProvider.java | 30 +++++---- .../singlefile/SingleFileImportProvider.java | 8 +-- .../runtime/cli/command/ImportRealmMixin.java | 8 +-- .../it/cli/dist/ImportAtStartupDistTest.java | 53 +++++++++++++-- .../exportimport/ExportImportConfig.java | 11 ++++ .../exportimport/ExportImportManager.java | 64 +++++++++++++++++++ .../resources/KeycloakApplication.java | 42 +++--------- 9 files changed, 211 insertions(+), 65 deletions(-) create mode 100644 model/legacy-services/src/main/java/org/keycloak/exportimport/AbstractFileBasedImportProvider.java diff --git a/docs/guides/src/main/server/importExport.adoc b/docs/guides/src/main/server/importExport.adoc index 15ce94e4c3..d106725fd0 100644 --- a/docs/guides/src/main/server/importExport.adoc +++ b/docs/guides/src/main/server/importExport.adoc @@ -79,12 +79,16 @@ You are also able to import realms when the server is starting by using the `--i <@kc.start parameters="--import-realm"/> -When you set the `--import-realm` option, the server is going to try to import any realm configuration file from the `data/import` directory. Each file in this directory should -contain a single realm configuration. Only regular files using the `.json` extension are read from this directory, sub-directories are ignored. +When you set the `--import-realm` option, the server is going to try to import any realm configuration file from the `data/import` directory. Only regular files using the `.json` extension are read from this directory, sub-directories are ignored. NOTE: For the https://quay.io/keycloak/keycloak[published containers], the import directory is `/opt/keycloak/data/import` -If a realm already exists in the server, the import operation is skipped. +If a realm already exists in the server, the import operation is skipped. The main reason behind this behavior is to avoid re-creating +realms and potentially loose state between server restarts. + +To re-create realms you should explicitly run the `import` command prior to starting the server. + +Importing the `master` realm is not supported because as it is a very sensitive operation. === Using Environment Variables within the Realm Configuration Files diff --git a/model/legacy-services/src/main/java/org/keycloak/exportimport/AbstractFileBasedImportProvider.java b/model/legacy-services/src/main/java/org/keycloak/exportimport/AbstractFileBasedImportProvider.java new file mode 100644 index 0000000000..1eb67d9d09 --- /dev/null +++ b/model/legacy-services/src/main/java/org/keycloak/exportimport/AbstractFileBasedImportProvider.java @@ -0,0 +1,50 @@ +/* + * Copyright 2022 Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.keycloak.exportimport; + +import static org.keycloak.common.util.StringPropertyReplacer.replaceProperties; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.util.Optional; +import org.keycloak.common.util.StringPropertyReplacer; + +public abstract class AbstractFileBasedImportProvider implements ImportProvider { + + private static final StringPropertyReplacer.PropertyResolver ENV_VAR_PROPERTY_RESOLVER = new StringPropertyReplacer.PropertyResolver() { + @Override + public String resolve(String property) { + return Optional.ofNullable(property).map(System::getenv).orElse(null); + } + }; + + protected InputStream parseFile(File importFile) throws IOException { + if (ExportImportConfig.isReplacePlaceholders()) { + String raw = new String(Files.readAllBytes(importFile.toPath()), "UTF-8"); + String parsed = replaceProperties(raw, ENV_VAR_PROPERTY_RESOLVER); + return new ByteArrayInputStream(parsed.getBytes()); + } + + return new FileInputStream(importFile); + } + +} diff --git a/model/legacy-services/src/main/java/org/keycloak/exportimport/dir/DirImportProvider.java b/model/legacy-services/src/main/java/org/keycloak/exportimport/dir/DirImportProvider.java index 56fd22c2f3..1f82b210c1 100755 --- a/model/legacy-services/src/main/java/org/keycloak/exportimport/dir/DirImportProvider.java +++ b/model/legacy-services/src/main/java/org/keycloak/exportimport/dir/DirImportProvider.java @@ -19,7 +19,7 @@ package org.keycloak.exportimport.dir; import org.jboss.logging.Logger; import org.keycloak.Config; -import org.keycloak.exportimport.ImportProvider; +import org.keycloak.exportimport.AbstractFileBasedImportProvider; import org.keycloak.exportimport.Strategy; import org.keycloak.exportimport.util.ExportImportSessionTask; import org.keycloak.exportimport.util.ImportUtils; @@ -31,9 +31,9 @@ import org.keycloak.representations.idm.RealmRepresentation; import org.keycloak.util.JsonSerialization; import java.io.File; -import java.io.FileInputStream; import java.io.FilenameFilter; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; @@ -42,7 +42,7 @@ import org.keycloak.services.managers.RealmManager; /** * @author Marek Posolda */ -public class DirImportProvider implements ImportProvider { +public class DirImportProvider extends AbstractFileBasedImportProvider { private static final Logger logger = Logger.getLogger(DirImportProvider.class); @@ -127,7 +127,7 @@ public class DirImportProvider implements ImportProvider { }); // Import realm first - FileInputStream is = new FileInputStream(realmFile); + InputStream is = parseFile(realmFile); final RealmRepresentation realmRep = JsonSerialization.readValue(is, RealmRepresentation.class); final AtomicBoolean realmImported = new AtomicBoolean(); @@ -144,7 +144,7 @@ public class DirImportProvider implements ImportProvider { if (realmImported.get()) { // Import users for (final File userFile : userFiles) { - final FileInputStream fis = new FileInputStream(userFile); + final InputStream fis = parseFile(userFile); KeycloakModelUtils.runJobInTransaction(factory, new ExportImportSessionTask() { @Override protected void runExportImportTask(KeycloakSession session) throws IOException { @@ -154,7 +154,7 @@ public class DirImportProvider implements ImportProvider { }); } for (final File userFile : federatedUserFiles) { - final FileInputStream fis = new FileInputStream(userFile); + final InputStream fis = parseFile(userFile); KeycloakModelUtils.runJobInTransaction(factory, new ExportImportSessionTask() { @Override protected void runExportImportTask(KeycloakSession session) throws IOException { @@ -165,16 +165,18 @@ public class DirImportProvider implements ImportProvider { } } - // Import authorization and initialize service accounts last, as they require users already in DB - KeycloakModelUtils.runJobInTransaction(factory, new ExportImportSessionTask() { + if (realmImported.get()) { + // Import authorization and initialize service accounts last, as they require users already in DB + KeycloakModelUtils.runJobInTransaction(factory, new ExportImportSessionTask() { - @Override - public void runExportImportTask(KeycloakSession session) throws IOException { - RealmManager realmManager = new RealmManager(session); - realmManager.setupClientServiceAccountsAndAuthorizationOnImport(realmRep, false); - } + @Override + public void runExportImportTask(KeycloakSession session) throws IOException { + RealmManager realmManager = new RealmManager(session); + realmManager.setupClientServiceAccountsAndAuthorizationOnImport(realmRep, false); + } - }); + }); + } } @Override diff --git a/model/legacy-services/src/main/java/org/keycloak/exportimport/singlefile/SingleFileImportProvider.java b/model/legacy-services/src/main/java/org/keycloak/exportimport/singlefile/SingleFileImportProvider.java index 856c6bddac..74c80c2763 100755 --- a/model/legacy-services/src/main/java/org/keycloak/exportimport/singlefile/SingleFileImportProvider.java +++ b/model/legacy-services/src/main/java/org/keycloak/exportimport/singlefile/SingleFileImportProvider.java @@ -19,7 +19,7 @@ package org.keycloak.exportimport.singlefile; import org.jboss.logging.Logger; import org.keycloak.Config; -import org.keycloak.exportimport.ImportProvider; +import org.keycloak.exportimport.AbstractFileBasedImportProvider; import org.keycloak.exportimport.Strategy; import org.keycloak.exportimport.util.ExportImportSessionTask; import org.keycloak.exportimport.util.ImportUtils; @@ -30,14 +30,14 @@ import org.keycloak.representations.idm.RealmRepresentation; import org.keycloak.util.JsonSerialization; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.util.Map; /** * @author Marek Posolda */ -public class SingleFileImportProvider implements ImportProvider { +public class SingleFileImportProvider extends AbstractFileBasedImportProvider { private static final Logger logger = Logger.getLogger(SingleFileImportProvider.class); @@ -73,7 +73,7 @@ public class SingleFileImportProvider implements ImportProvider { protected void checkRealmReps() throws IOException { if (realmReps == null) { - FileInputStream is = new FileInputStream(file); + InputStream is = parseFile(file); realmReps = ImportUtils.getRealmsFromStream(JsonSerialization.mapper, is); } } diff --git a/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/cli/command/ImportRealmMixin.java b/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/cli/command/ImportRealmMixin.java index 646007481b..933a6a8177 100644 --- a/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/cli/command/ImportRealmMixin.java +++ b/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/cli/command/ImportRealmMixin.java @@ -40,13 +40,7 @@ public final class ImportRealmMixin { File importDir = Environment.getHomePath().resolve("data").resolve("import").toFile(); if (importDir.exists()) { - StringBuilder filesToImport = new StringBuilder(); - - for (File realmFile : importDir.listFiles()) { - filesToImport.append(realmFile.getAbsolutePath()).append(","); - } - - System.setProperty("keycloak.import", filesToImport.toString()); + System.setProperty("keycloak.import", importDir.getAbsolutePath()); } } } diff --git a/quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/ImportAtStartupDistTest.java b/quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/ImportAtStartupDistTest.java index 882146a71f..fcea519dc9 100644 --- a/quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/ImportAtStartupDistTest.java +++ b/quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/ImportAtStartupDistTest.java @@ -17,6 +17,7 @@ package org.keycloak.it.cli.dist; +import java.io.IOException; import java.nio.file.Path; import java.util.function.Consumer; import org.junit.jupiter.api.Test; @@ -29,6 +30,7 @@ import org.keycloak.it.junit5.extension.RawDistOnly; import org.keycloak.it.utils.KeycloakDistribution; import org.keycloak.it.utils.RawKeycloakDistribution; +import io.quarkus.deployment.util.FileUtil; import io.quarkus.test.junit.main.Launch; import io.quarkus.test.junit.main.LaunchResult; @@ -41,21 +43,21 @@ public class ImportAtStartupDistTest { @Launch({"start-dev", "--import-realm"}) void testImport(LaunchResult result) { CLIResult cliResult = (CLIResult) result; - cliResult.assertMessage("Imported realm quickstart-realm from file"); + cliResult.assertMessage("Realm 'quickstart-realm' imported"); } @Test @BeforeStartDistribution(CreateRealmConfigurationFileAndDir.class) - @Launch({"start-dev", "--import-realm", "--log-level=org.keycloak.services.resources.KeycloakApplication:debug"}) + @Launch({"start-dev", "--import-realm", "--log-level=org.keycloak.exportimport.ExportImportManager:debug"}) void testImportAndIgnoreDirectory(LaunchResult result) { CLIResult cliResult = (CLIResult) result; - cliResult.assertMessage("Imported realm quickstart-realm from file"); + cliResult.assertMessage("Realm 'quickstart-realm' imported"); cliResult.assertMessage("Ignoring import file because it is not a valid file"); } @Test @BeforeStartDistribution(CreateRealmConfigurationFileWithUnsupportedExtension.class) - @Launch({"start-dev", "--import-realm", "--log-level=org.keycloak.services.resources.KeycloakApplication:debug"}) + @Launch({"start-dev", "--import-realm", "--log-level=org.keycloak.exportimport.ExportImportManager:debug"}) void testIgnoreFileWithUnsupportedExtension(LaunchResult result) { CLIResult cliResult = (CLIResult) result; cliResult.assertMessage("Ignoring import file because it is not a valid file"); @@ -70,6 +72,49 @@ public class ImportAtStartupDistTest { cliResult.assertError("option '--import-realm' should be specified without 'some-file' parameter"); } + @Test + @BeforeStartDistribution(CreateRealmConfigurationFile.class) + void testImportFromFileCreatedByExportAllRealms(KeycloakDistribution dist) throws IOException { + dist.run("start-dev", "--import-realm"); + dist.run("export", "--file=../data/import/realm.json"); + + RawKeycloakDistribution rawDist = dist.unwrap(RawKeycloakDistribution.class); + FileUtil.deleteDirectory(rawDist.getDistPath().resolve("data").resolve("chm").toAbsolutePath()); + + CLIResult result = dist.run("start-dev", "--import-realm"); + result.assertMessage("Realm 'quickstart-realm' imported"); + result.assertMessage("Realm 'master' already exists. Import skipped"); + } + + @Test + @BeforeStartDistribution(CreateRealmConfigurationFile.class) + void testImportFromFileCreatedByExportSingleRealm(KeycloakDistribution dist) throws IOException { + dist.run("start-dev", "--import-realm"); + dist.run("export", "--realm=quickstart-realm", "--file=../data/import/realm.json"); + + RawKeycloakDistribution rawDist = dist.unwrap(RawKeycloakDistribution.class); + FileUtil.deleteDirectory(rawDist.getDistPath().resolve("data").resolve("chm").toAbsolutePath()); + + CLIResult result = dist.run("start-dev", "--import-realm"); + result.assertMessage("Realm 'quickstart-realm' imported"); + result.assertNoMessage("Not importing realm master from file"); + } + + @Test + @BeforeStartDistribution(CreateRealmConfigurationFile.class) + void testImportFromDirCreatedByExport(KeycloakDistribution dist) throws IOException { + dist.run("start-dev", "--import-realm"); + RawKeycloakDistribution rawDist = dist.unwrap(RawKeycloakDistribution.class); + FileUtil.deleteDirectory(rawDist.getDistPath().resolve("data").resolve("import").toAbsolutePath()); + dist.run("export", "--dir=../data/import"); + + FileUtil.deleteDirectory(rawDist.getDistPath().resolve("data").resolve("chm").toAbsolutePath()); + + CLIResult result = dist.run("start-dev", "--import-realm"); + result.assertMessage("Realm 'quickstart-realm' imported"); + result.assertNoMessage("Not importing realm master from file"); + } + public static class CreateRealmConfigurationFile implements Consumer { @Override diff --git a/services/src/main/java/org/keycloak/exportimport/ExportImportConfig.java b/services/src/main/java/org/keycloak/exportimport/ExportImportConfig.java index 2555a19da7..77693faefc 100644 --- a/services/src/main/java/org/keycloak/exportimport/ExportImportConfig.java +++ b/services/src/main/java/org/keycloak/exportimport/ExportImportConfig.java @@ -39,6 +39,9 @@ public class ExportImportConfig { // used for "singleFile" provider public static final String FILE = PREFIX + "file"; + // used for replacing placeholders + public static final String REPLACE_PLACEHOLDERS = PREFIX + "replace-placeholders"; + // How to export users when realm export is requested for "dir" provider public static final String USERS_EXPORT_STRATEGY = PREFIX + "usersExportStrategy"; public static final UsersExportStrategy DEFAULT_USERS_EXPORT_STRATEGY = UsersExportStrategy.DIFFERENT_FILES; @@ -117,4 +120,12 @@ public class ExportImportConfig { String strategy = System.getProperty(STRATEGY, DEFAULT_STRATEGY.toString()); return Enum.valueOf(Strategy.class, strategy); } + + public static boolean isReplacePlaceholders() { + return Boolean.getBoolean(REPLACE_PLACEHOLDERS); + } + + public static void setReplacePlaceholders(boolean replacePlaceholders) { + System.setProperty(REPLACE_PLACEHOLDERS, String.valueOf(replacePlaceholders)); + } } diff --git a/services/src/main/java/org/keycloak/exportimport/ExportImportManager.java b/services/src/main/java/org/keycloak/exportimport/ExportImportManager.java index 3e2188e549..5bf1a87403 100644 --- a/services/src/main/java/org/keycloak/exportimport/ExportImportManager.java +++ b/services/src/main/java/org/keycloak/exportimport/ExportImportManager.java @@ -21,9 +21,20 @@ package org.keycloak.exportimport; import org.jboss.logging.Logger; import org.keycloak.models.KeycloakSession; import org.keycloak.models.KeycloakSessionFactory; +import org.keycloak.models.KeycloakSessionTask; +import org.keycloak.models.utils.KeycloakModelUtils; +import org.keycloak.provider.ProviderFactory; import org.keycloak.services.ServicesLogger; +import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * @author Marek Posolda @@ -33,6 +44,7 @@ public class ExportImportManager { private static final Logger logger = Logger.getLogger(ExportImportManager.class); private KeycloakSessionFactory sessionFactory; + private KeycloakSession session; private final String realmName; @@ -41,6 +53,7 @@ public class ExportImportManager { public ExportImportManager(KeycloakSession session) { this.sessionFactory = session.getKeycloakSessionFactory(); + this.session = session; realmName = ExportImportConfig.getRealmName(); @@ -95,6 +108,57 @@ public class ExportImportManager { } } + public void runImportAtStartup(String dir, Strategy strategy) throws IOException { + ExportImportConfig.setReplacePlaceholders(true); + ExportImportConfig.setAction("import"); + + Stream factories = sessionFactory.getProviderFactoriesStream(ImportProvider.class); + + for (ProviderFactory factory : factories.collect(Collectors.toList())) { + String providerId = factory.getId(); + + if ("dir".equals(providerId)) { + ExportImportConfig.setDir(dir); + ImportProvider importProvider = session.getProvider(ImportProvider.class, providerId); + importProvider.importModel(sessionFactory, strategy); + } else if ("singleFile".equals(providerId)) { + Set filesToImport = new HashSet<>(); + + for (File file : Paths.get(dir).toFile().listFiles()) { + Path filePath = file.toPath(); + + if (!(Files.exists(filePath) && Files.isRegularFile(filePath) && filePath.toString().endsWith(".json"))) { + logger.debugf("Ignoring import file because it is not a valid file: %s", file); + continue; + } + + String fileName = file.getName(); + + if (fileName.contains("-realm.json") || fileName.contains("-users-")) { + continue; + } + + filesToImport.add(file.getAbsolutePath()); + } + + for (String file : filesToImport) { + ExportImportConfig.setFile(file); + KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() { + @Override + public void run(KeycloakSession session) { + ImportProvider importProvider = session.getProvider(ImportProvider.class, providerId); + try { + importProvider.importModel(sessionFactory, strategy); + } catch (IOException cause) { + throw new RuntimeException(cause); + } + } + }); + } + } + } + } + public void runExport() { try { if (realmName == null) { diff --git a/services/src/main/java/org/keycloak/services/resources/KeycloakApplication.java b/services/src/main/java/org/keycloak/services/resources/KeycloakApplication.java index 60bbebde7f..4269e29744 100644 --- a/services/src/main/java/org/keycloak/services/resources/KeycloakApplication.java +++ b/services/src/main/java/org/keycloak/services/resources/KeycloakApplication.java @@ -22,9 +22,9 @@ import org.keycloak.Config; import org.keycloak.common.Profile; import org.keycloak.common.crypto.CryptoIntegration; import org.keycloak.common.util.Resteasy; -import org.keycloak.common.util.StringPropertyReplacer; import org.keycloak.config.ConfigProviderFactory; import org.keycloak.exportimport.ExportImportManager; +import org.keycloak.exportimport.Strategy; import org.keycloak.models.KeycloakSession; import org.keycloak.models.KeycloakSessionFactory; import org.keycloak.models.KeycloakSessionTask; @@ -60,16 +60,11 @@ import javax.ws.rs.core.Application; import java.io.File; import java.io.FileInputStream; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.HashSet; import java.util.List; import java.util.NoSuchElementException; -import java.util.Optional; import java.util.ServiceLoader; import java.util.Set; -import java.util.StringTokenizer; import java.util.concurrent.atomic.AtomicBoolean; /** @@ -220,7 +215,7 @@ public class KeycloakApplication extends Application { if (exportImportManager[0].isRunImport()) { exportImportManager[0].runImport(); } else { - importRealms(); + importRealms(exportImportManager[0]); } importAddUser(); @@ -262,32 +257,13 @@ public class KeycloakApplication extends Application { return singletons; } - public void importRealms() { - String files = System.getProperty("keycloak.import"); - if (files != null) { - StringTokenizer tokenizer = new StringTokenizer(files, ","); - while (tokenizer.hasMoreTokens()) { - String file = tokenizer.nextToken().trim(); - RealmRepresentation rep; - try { - Path filePath = Paths.get(file); - - if (!(Files.exists(filePath) && Files.isRegularFile(filePath) && filePath.toString().endsWith(".json"))) { - logger.debugf("Ignoring import file because it is not a valid file: %s", file); - continue; - } - - rep = JsonSerialization.readValue(StringPropertyReplacer.replaceProperties( - new String(Files.readAllBytes(filePath), "UTF-8"), new StringPropertyReplacer.PropertyResolver() { - @Override - public String resolve(String property) { - return Optional.ofNullable(System.getenv(property)).orElse(null); - } - }), RealmRepresentation.class); - } catch (Exception cause) { - throw new RuntimeException("Failed to parse realm configuration file: " + file, cause); - } - importRealm(rep, "file " + file); + public void importRealms(ExportImportManager exportImportManager) { + String dir = System.getProperty("keycloak.import"); + if (dir != null) { + try { + exportImportManager.runImportAtStartup(dir, Strategy.IGNORE_EXISTING); + } catch (IOException cause) { + throw new RuntimeException("Failed to import realms", cause); } } }