libre.sh/keycloak-scim
3
0
Fork 0
Code Issues 15 Pull requests Releases Packages Activity Actions

Keep consistency when importing realms at startup when they are exported via the export command

Browse source 
Closes #16281

Co-authored-by: Martin Bartoš <mabartos@redhat.com>
This commit is contained in:
Pedro Igor 2023-01-05 14:31:21 -03:00 committed by Václav Muzikář
parent 53ee95764e
commit 522bf1c0b0
9 changed files with 211 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
docs/guides/src/main/server/importExport.adoc
View file

@ -79,12 +79,16 @@ You are also able to import realms when the server is starting by using the `--i
<@kc.start parameters="--import-realm"/>
When you set the `--import-realm` option, the server is going to try to import any realm configuration file from the `data/import` directory. Each file in this directory should
contain a single realm configuration. Only regular files using the `.json` extension are read from this directory, sub-directories are ignored.
When you set the `--import-realm` option, the server is going to try to import any realm configuration file from the `data/import` directory. Only regular files using the `.json` extension are read from this directory, sub-directories are ignored.
NOTE: For the https://quay.io/keycloak/keycloak[published containers], the import directory is `/opt/keycloak/data/import`
If a realm already exists in the server, the import operation is skipped.
If a realm already exists in the server, the import operation is skipped. The main reason behind this behavior is to avoid re-creating
realms and potentially loose state between server restarts.
To re-create realms you should explicitly run the `import` command prior to starting the server.
Importing the `master` realm is not supported because as it is a very sensitive operation.
=== Using Environment Variables within the Realm Configuration Files

50
model/legacy-services/src/main/java/org/keycloak/exportimport/AbstractFileBasedImportProvider.java Normal file
View file

@ -0,0 +1,50 @@
/*
* Copyright 2022 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.exportimport;
import static org.keycloak.common.util.StringPropertyReplacer.replaceProperties;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.util.Optional;
import org.keycloak.common.util.StringPropertyReplacer;
public abstract class AbstractFileBasedImportProvider implements ImportProvider {
private static final StringPropertyReplacer.PropertyResolver ENV_VAR_PROPERTY_RESOLVER = new StringPropertyReplacer.PropertyResolver() {
@Override
public String resolve(String property) {
return Optional.ofNullable(property).map(System::getenv).orElse(null);
}
};
protected InputStream parseFile(File importFile) throws IOException {
if (ExportImportConfig.isReplacePlaceholders()) {
String raw = new String(Files.readAllBytes(importFile.toPath()), "UTF-8");
String parsed = replaceProperties(raw, ENV_VAR_PROPERTY_RESOLVER);
return new ByteArrayInputStream(parsed.getBytes());
}
return new FileInputStream(importFile);
}
}

14
model/legacy-services/src/main/java/org/keycloak/exportimport/dir/DirImportProvider.java
View file

@ -19,7 +19,7 @@ package org.keycloak.exportimport.dir;
import org.jboss.logging.Logger;
import org.keycloak.Config;
import org.keycloak.exportimport.ImportProvider;
import org.keycloak.exportimport.AbstractFileBasedImportProvider;
import org.keycloak.exportimport.Strategy;
import org.keycloak.exportimport.util.ExportImportSessionTask;
import org.keycloak.exportimport.util.ImportUtils;
@ -31,9 +31,9 @@ import org.keycloak.representations.idm.RealmRepresentation;
import org.keycloak.util.JsonSerialization;
import java.io.File;
import java.io.FileInputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
@ -42,7 +42,7 @@ import org.keycloak.services.managers.RealmManager;
/**
* @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
*/
public class DirImportProvider implements ImportProvider {
public class DirImportProvider extends AbstractFileBasedImportProvider {
private static final Logger logger = Logger.getLogger(DirImportProvider.class);
@ -127,7 +127,7 @@ public class DirImportProvider implements ImportProvider {
});
// Import realm first
FileInputStream is = new FileInputStream(realmFile);
InputStream is = parseFile(realmFile);
final RealmRepresentation realmRep = JsonSerialization.readValue(is, RealmRepresentation.class);
final AtomicBoolean realmImported = new AtomicBoolean();
@ -144,7 +144,7 @@ public class DirImportProvider implements ImportProvider {
if (realmImported.get()) {
// Import users
for (final File userFile : userFiles) {
final FileInputStream fis = new FileInputStream(userFile);
final InputStream fis = parseFile(userFile);
KeycloakModelUtils.runJobInTransaction(factory, new ExportImportSessionTask() {
@Override
protected void runExportImportTask(KeycloakSession session) throws IOException {
@ -154,7 +154,7 @@ public class DirImportProvider implements ImportProvider {
});
}
for (final File userFile : federatedUserFiles) {
final FileInputStream fis = new FileInputStream(userFile);
final InputStream fis = parseFile(userFile);
KeycloakModelUtils.runJobInTransaction(factory, new ExportImportSessionTask() {
@Override
protected void runExportImportTask(KeycloakSession session) throws IOException {
@ -165,6 +165,7 @@ public class DirImportProvider implements ImportProvider {
}
}
if (realmImported.get()) {
// Import authorization and initialize service accounts last, as they require users already in DB
KeycloakModelUtils.runJobInTransaction(factory, new ExportImportSessionTask() {
@ -176,6 +177,7 @@ public class DirImportProvider implements ImportProvider {
});
}
}
@Override
public void close() {

8
model/legacy-services/src/main/java/org/keycloak/exportimport/singlefile/SingleFileImportProvider.java
View file

@ -19,7 +19,7 @@ package org.keycloak.exportimport.singlefile;
import org.jboss.logging.Logger;
import org.keycloak.Config;
import org.keycloak.exportimport.ImportProvider;
import org.keycloak.exportimport.AbstractFileBasedImportProvider;
import org.keycloak.exportimport.Strategy;
import org.keycloak.exportimport.util.ExportImportSessionTask;
import org.keycloak.exportimport.util.ImportUtils;
@ -30,14 +30,14 @@ import org.keycloak.representations.idm.RealmRepresentation;
import org.keycloak.util.JsonSerialization;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Map;
/**
* @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
*/
public class SingleFileImportProvider implements ImportProvider {
public class SingleFileImportProvider extends AbstractFileBasedImportProvider {
private static final Logger logger = Logger.getLogger(SingleFileImportProvider.class);
@ -73,7 +73,7 @@ public class SingleFileImportProvider implements ImportProvider {
protected void checkRealmReps() throws IOException {
if (realmReps == null) {
FileInputStream is = new FileInputStream(file);
InputStream is = parseFile(file);
realmReps = ImportUtils.getRealmsFromStream(JsonSerialization.mapper, is);
}
}

8
quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/cli/command/ImportRealmMixin.java
View file

@ -40,13 +40,7 @@ public final class ImportRealmMixin {
File importDir = Environment.getHomePath().resolve("data").resolve("import").toFile();
if (importDir.exists()) {
StringBuilder filesToImport = new StringBuilder();
for (File realmFile : importDir.listFiles()) {
filesToImport.append(realmFile.getAbsolutePath()).append(",");
}
System.setProperty("keycloak.import", filesToImport.toString());
System.setProperty("keycloak.import", importDir.getAbsolutePath());
}
}
}

53
quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/ImportAtStartupDistTest.java vendored
View file

@ -17,6 +17,7 @@
package org.keycloak.it.cli.dist;
import java.io.IOException;
import java.nio.file.Path;
import java.util.function.Consumer;
import org.junit.jupiter.api.Test;
@ -29,6 +30,7 @@ import org.keycloak.it.junit5.extension.RawDistOnly;
import org.keycloak.it.utils.KeycloakDistribution;
import org.keycloak.it.utils.RawKeycloakDistribution;
import io.quarkus.deployment.util.FileUtil;
import io.quarkus.test.junit.main.Launch;
import io.quarkus.test.junit.main.LaunchResult;
@ -41,21 +43,21 @@ public class ImportAtStartupDistTest {
@Launch({"start-dev", "--import-realm"})
void testImport(LaunchResult result) {
CLIResult cliResult = (CLIResult) result;
cliResult.assertMessage("Imported realm quickstart-realm from file");
cliResult.assertMessage("Realm 'quickstart-realm' imported");
}
@Test
@BeforeStartDistribution(CreateRealmConfigurationFileAndDir.class)
@Launch({"start-dev", "--import-realm", "--log-level=org.keycloak.services.resources.KeycloakApplication:debug"})
@Launch({"start-dev", "--import-realm", "--log-level=org.keycloak.exportimport.ExportImportManager:debug"})
void testImportAndIgnoreDirectory(LaunchResult result) {
CLIResult cliResult = (CLIResult) result;
cliResult.assertMessage("Imported realm quickstart-realm from file");
cliResult.assertMessage("Realm 'quickstart-realm' imported");
cliResult.assertMessage("Ignoring import file because it is not a valid file");
}
@Test
@BeforeStartDistribution(CreateRealmConfigurationFileWithUnsupportedExtension.class)
@Launch({"start-dev", "--import-realm", "--log-level=org.keycloak.services.resources.KeycloakApplication:debug"})
@Launch({"start-dev", "--import-realm", "--log-level=org.keycloak.exportimport.ExportImportManager:debug"})
void testIgnoreFileWithUnsupportedExtension(LaunchResult result) {
CLIResult cliResult = (CLIResult) result;
cliResult.assertMessage("Ignoring import file because it is not a valid file");
@ -70,6 +72,49 @@ public class ImportAtStartupDistTest {
cliResult.assertError("option '--import-realm' should be specified without 'some-file' parameter");
}
@Test
@BeforeStartDistribution(CreateRealmConfigurationFile.class)
void testImportFromFileCreatedByExportAllRealms(KeycloakDistribution dist) throws IOException {
dist.run("start-dev", "--import-realm");
dist.run("export", "--file=../data/import/realm.json");
RawKeycloakDistribution rawDist = dist.unwrap(RawKeycloakDistribution.class);
FileUtil.deleteDirectory(rawDist.getDistPath().resolve("data").resolve("chm").toAbsolutePath());
CLIResult result = dist.run("start-dev", "--import-realm");
result.assertMessage("Realm 'quickstart-realm' imported");
result.assertMessage("Realm 'master' already exists. Import skipped");
}
@Test
@BeforeStartDistribution(CreateRealmConfigurationFile.class)
void testImportFromFileCreatedByExportSingleRealm(KeycloakDistribution dist) throws IOException {
dist.run("start-dev", "--import-realm");
dist.run("export", "--realm=quickstart-realm", "--file=../data/import/realm.json");
RawKeycloakDistribution rawDist = dist.unwrap(RawKeycloakDistribution.class);
FileUtil.deleteDirectory(rawDist.getDistPath().resolve("data").resolve("chm").toAbsolutePath());
CLIResult result = dist.run("start-dev", "--import-realm");
result.assertMessage("Realm 'quickstart-realm' imported");
result.assertNoMessage("Not importing realm master from file");
}
@Test
@BeforeStartDistribution(CreateRealmConfigurationFile.class)
void testImportFromDirCreatedByExport(KeycloakDistribution dist) throws IOException {
dist.run("start-dev", "--import-realm");
RawKeycloakDistribution rawDist = dist.unwrap(RawKeycloakDistribution.class);
FileUtil.deleteDirectory(rawDist.getDistPath().resolve("data").resolve("import").toAbsolutePath());
dist.run("export", "--dir=../data/import");
FileUtil.deleteDirectory(rawDist.getDistPath().resolve("data").resolve("chm").toAbsolutePath());
CLIResult result = dist.run("start-dev", "--import-realm");
result.assertMessage("Realm 'quickstart-realm' imported");
result.assertNoMessage("Not importing realm master from file");
}
public static class CreateRealmConfigurationFile implements Consumer<KeycloakDistribution> {
@Override

11
services/src/main/java/org/keycloak/exportimport/ExportImportConfig.java
View file

@ -39,6 +39,9 @@ public class ExportImportConfig {
// used for "singleFile" provider
public static final String FILE = PREFIX + "file";
// used for replacing placeholders
public static final String REPLACE_PLACEHOLDERS = PREFIX + "replace-placeholders";
// How to export users when realm export is requested for "dir" provider
public static final String USERS_EXPORT_STRATEGY = PREFIX + "usersExportStrategy";
public static final UsersExportStrategy DEFAULT_USERS_EXPORT_STRATEGY = UsersExportStrategy.DIFFERENT_FILES;
@ -117,4 +120,12 @@ public class ExportImportConfig {
String strategy = System.getProperty(STRATEGY, DEFAULT_STRATEGY.toString());
return Enum.valueOf(Strategy.class, strategy);
}
public static boolean isReplacePlaceholders() {
return Boolean.getBoolean(REPLACE_PLACEHOLDERS);
}
public static void setReplacePlaceholders(boolean replacePlaceholders) {
System.setProperty(REPLACE_PLACEHOLDERS, String.valueOf(replacePlaceholders));
}
}

64
services/src/main/java/org/keycloak/exportimport/ExportImportManager.java
View file

@ -21,9 +21,20 @@ package org.keycloak.exportimport;
import org.jboss.logging.Logger;
import org.keycloak.models.KeycloakSession;
import org.keycloak.models.KeycloakSessionFactory;
import org.keycloak.models.KeycloakSessionTask;
import org.keycloak.models.utils.KeycloakModelUtils;
import org.keycloak.provider.ProviderFactory;
import org.keycloak.services.ServicesLogger;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashSet;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
@ -33,6 +44,7 @@ public class ExportImportManager {
private static final Logger logger = Logger.getLogger(ExportImportManager.class);
private KeycloakSessionFactory sessionFactory;
private KeycloakSession session;
private final String realmName;
@ -41,6 +53,7 @@ public class ExportImportManager {
public ExportImportManager(KeycloakSession session) {
this.sessionFactory = session.getKeycloakSessionFactory();
this.session = session;
realmName = ExportImportConfig.getRealmName();
@ -95,6 +108,57 @@ public class ExportImportManager {
}
}
public void runImportAtStartup(String dir, Strategy strategy) throws IOException {
ExportImportConfig.setReplacePlaceholders(true);
ExportImportConfig.setAction("import");
Stream<ProviderFactory> factories = sessionFactory.getProviderFactoriesStream(ImportProvider.class);
for (ProviderFactory factory : factories.collect(Collectors.toList())) {
String providerId = factory.getId();
if ("dir".equals(providerId)) {
ExportImportConfig.setDir(dir);
ImportProvider importProvider = session.getProvider(ImportProvider.class, providerId);
importProvider.importModel(sessionFactory, strategy);
} else if ("singleFile".equals(providerId)) {
Set<String> filesToImport = new HashSet<>();
for (File file : Paths.get(dir).toFile().listFiles()) {
Path filePath = file.toPath();
if (!(Files.exists(filePath) && Files.isRegularFile(filePath) && filePath.toString().endsWith(".json"))) {
logger.debugf("Ignoring import file because it is not a valid file: %s", file);
continue;
}
String fileName = file.getName();
if (fileName.contains("-realm.json") || fileName.contains("-users-")) {
continue;
}
filesToImport.add(file.getAbsolutePath());
}
for (String file : filesToImport) {
ExportImportConfig.setFile(file);
KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
@Override
public void run(KeycloakSession session) {
ImportProvider importProvider = session.getProvider(ImportProvider.class, providerId);
try {
importProvider.importModel(sessionFactory, strategy);
} catch (IOException cause) {
throw new RuntimeException(cause);
}
}
});
}
}
}
}
public void runExport() {
try {
if (realmName == null) {

40
services/src/main/java/org/keycloak/services/resources/KeycloakApplication.java
View file

@ -22,9 +22,9 @@ import org.keycloak.Config;
import org.keycloak.common.Profile;
import org.keycloak.common.crypto.CryptoIntegration;
import org.keycloak.common.util.Resteasy;
import org.keycloak.common.util.StringPropertyReplacer;
import org.keycloak.config.ConfigProviderFactory;
import org.keycloak.exportimport.ExportImportManager;
import org.keycloak.exportimport.Strategy;
import org.keycloak.models.KeycloakSession;
import org.keycloak.models.KeycloakSessionFactory;
import org.keycloak.models.KeycloakSessionTask;
@ -60,16 +60,11 @@ import javax.ws.rs.core.Application;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashSet;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Optional;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.concurrent.atomic.AtomicBoolean;
/**
@ -220,7 +215,7 @@ public class KeycloakApplication extends Application {
if (exportImportManager[0].isRunImport()) {
exportImportManager[0].runImport();
} else {
importRealms();
importRealms(exportImportManager[0]);
}
importAddUser();
@ -262,32 +257,13 @@ public class KeycloakApplication extends Application {
return singletons;
}
public void importRealms() {
String files = System.getProperty("keycloak.import");
if (files != null) {
StringTokenizer tokenizer = new StringTokenizer(files, ",");
while (tokenizer.hasMoreTokens()) {
String file = tokenizer.nextToken().trim();
RealmRepresentation rep;
public void importRealms(ExportImportManager exportImportManager) {
String dir = System.getProperty("keycloak.import");
if (dir != null) {
try {
Path filePath = Paths.get(file);
if (!(Files.exists(filePath) && Files.isRegularFile(filePath) && filePath.toString().endsWith(".json"))) {
logger.debugf("Ignoring import file because it is not a valid file: %s", file);
continue;
}
rep = JsonSerialization.readValue(StringPropertyReplacer.replaceProperties(
new String(Files.readAllBytes(filePath), "UTF-8"), new StringPropertyReplacer.PropertyResolver() {
@Override
public String resolve(String property) {
return Optional.ofNullable(System.getenv(property)).orElse(null);
}
}), RealmRepresentation.class);
} catch (Exception cause) {
throw new RuntimeException("Failed to parse realm configuration file: " + file, cause);
}
importRealm(rep, "file " + file);
exportImportManager.runImportAtStartup(dir, Strategy.IGNORE_EXISTING);
} catch (IOException cause) {
throw new RuntimeException("Failed to import realms", cause);
}
}
}