From b6b6d01a8a1941c0cc98b0a12f267c83958c6632 Mon Sep 17 00:00:00 2001 From: Alexander Schwartz Date: Tue, 27 Sep 2022 17:13:02 +0200 Subject: [PATCH] Importing a representation by first creating the defaults, importing a representation and then copying it over to the real store. This is the foundation for a setup that's needed when importing the new file store for which importing the representation serves as a placeholder. Closes #14583 --- ...enerateEntityImplementationsProcessor.java | 49 ++- .../jpa/JpaMapKeycloakTransaction.java | 2 +- .../role/JpaRoleMapKeycloakTransaction.java | 15 + .../delegate/JpaMapRoleEntityDelegate.java | 18 +- .../common/AbstractMapProviderFactory.java | 2 +- .../map/datastore/ImportKeycloakSession.java | 373 ++++++++++++++++++ .../ImportSessionFactoryWrapper.java | 128 ++++++ .../map/datastore/MapExportImportManager.java | 308 +++++++++++++-- .../storage/SetDefaultsForNewRealm.java | 51 +++ .../keycloak/utils/ReservedCharValidator.java | 0 .../services/managers/RealmManager.java | 20 + .../managers/RealmManagerProviderFactory.java | 4 + 12 files changed, 927 insertions(+), 43 deletions(-) create mode 100644 model/map/src/main/java/org/keycloak/models/map/datastore/ImportKeycloakSession.java create mode 100644 model/map/src/main/java/org/keycloak/models/map/datastore/ImportSessionFactoryWrapper.java create mode 100644 server-spi-private/src/main/java/org/keycloak/storage/SetDefaultsForNewRealm.java rename {services => server-spi-private}/src/main/java/org/keycloak/utils/ReservedCharValidator.java (100%) diff --git a/model/build-processor/src/main/java/org/keycloak/models/map/processor/GenerateEntityImplementationsProcessor.java b/model/build-processor/src/main/java/org/keycloak/models/map/processor/GenerateEntityImplementationsProcessor.java index 5fbc662f3d..a709908dc1 100644 --- a/model/build-processor/src/main/java/org/keycloak/models/map/processor/GenerateEntityImplementationsProcessor.java +++ b/model/build-processor/src/main/java/org/keycloak/models/map/processor/GenerateEntityImplementationsProcessor.java @@ -60,6 +60,7 @@ import javax.lang.model.type.TypeKind; public class GenerateEntityImplementationsProcessor extends AbstractGenerateEntityImplementationsProcessor { private static final Collection autogenerated = new TreeSet<>(); + private static final String ID_FIELD_NAME = "Id"; private final Generator[] generators = new Generator[] { new ClonerGenerator(), @@ -260,7 +261,7 @@ public class GenerateEntityImplementationsProcessor extends AbstractGenerateEnti String simpleClassName = className.substring(lastDot + 1); String mapImplClassName = className + "Impl"; String mapSimpleClassName = simpleClassName + "Impl"; - boolean hasId = methodsPerAttribute.containsKey("Id") || allParentMembers.stream().anyMatch(el -> "getId".equals(el.getSimpleName().toString())); + boolean hasId = methodsPerAttribute.containsKey(ID_FIELD_NAME) || allParentMembers.stream().anyMatch(el -> "getId".equals(el.getSimpleName().toString())); boolean hasDeepClone = allParentMembers.stream().filter(el -> el.getKind() == ElementKind.METHOD).anyMatch(el -> "deepClone".equals(el.getSimpleName().toString())); boolean needsDeepClone = fieldGetters(methodsPerAttribute) .map(ExecutableElement::getReturnType) @@ -706,32 +707,28 @@ public class GenerateEntityImplementationsProcessor extends AbstractGenerateEnti pw.println("import " + FQN_DEEP_CLONER + ";"); pw.println("// DO NOT CHANGE THIS CLASS, IT IS GENERATED AUTOMATICALLY BY " + GenerateEntityImplementationsProcessor.class.getSimpleName()); pw.println("public class " + clonerSimpleClassName + " {"); - pw.println(" public static " + className + " deepClone(" + className + " original, " + className + " target) {"); - methodsPerAttribute.entrySet().stream().sorted(Comparator.comparing(Map.Entry::getKey)).forEach(me -> { - final String fieldName = me.getKey(); - HashSet methods = me.getValue(); - TypeMirror fieldType = determineFieldType(fieldName, methods); - if (fieldType == null) { - return; - } + if (methodsPerAttribute.containsKey(ID_FIELD_NAME)) { + pw.println(" public static " + className + " deepClone(" + className + " original, " + className + " target) {"); - cloneField(e, fieldName, methods, fieldType, pw); - }); - pw.println(" target.clearUpdatedFlag();"); - pw.println(" return target;"); - pw.println(" }"); + // If the entity has an ID, set the ID first and then set all other attributes. + // This was important when working with Jpa storage as the ID is the one field needed to persist an entity. + HashSet idMethods = methodsPerAttribute.get(ID_FIELD_NAME); + TypeMirror idFieldType = determineFieldType(ID_FIELD_NAME, idMethods); + cloneField(e, ID_FIELD_NAME, idMethods, idFieldType, pw); - autogenerated.add(" CLONERS_WITH_ID.put(" + className + ".class, (Cloner<" + className + ">) " + clonerImplClassName + "::deepClone);"); + pw.println(" return deepCloneNoId(original, target);"); + pw.println(" }"); + + autogenerated.add(" CLONERS_WITH_ID.put(" + className + ".class, (Cloner<" + className + ">) " + clonerImplClassName + "::deepClone);"); - if (methodsPerAttribute.containsKey("Id")) { pw.println(" public static " + className + " deepCloneNoId(" + className + " original, " + className + " target) {"); methodsPerAttribute.entrySet().stream().sorted(Comparator.comparing(Map.Entry::getKey)).forEach(me -> { final String fieldName = me.getKey(); HashSet methods = me.getValue(); TypeMirror fieldType = determineFieldType(fieldName, methods); - if (fieldType == null || "Id".equals(fieldName)) { + if (fieldType == null || ID_FIELD_NAME.equals(fieldName)) { return; } @@ -742,6 +739,24 @@ public class GenerateEntityImplementationsProcessor extends AbstractGenerateEnti pw.println(" }"); autogenerated.add(" CLONERS_WITHOUT_ID.put(" + className + ".class, (Cloner<" + className + ">) " + clonerImplClassName + "::deepCloneNoId);"); + } else { + pw.println(" public static " + className + " deepClone(" + className + " original, " + className + " target) {"); + + methodsPerAttribute.entrySet().stream().sorted(Comparator.comparing(Map.Entry::getKey)).forEach(me -> { + final String fieldName = me.getKey(); + HashSet methods = me.getValue(); + TypeMirror fieldType = determineFieldType(fieldName, methods); + if (fieldType == null) { + return; + } + + cloneField(e, fieldName, methods, fieldType, pw); + }); + pw.println(" target.clearUpdatedFlag();"); + pw.println(" return target;"); + pw.println(" }"); + + autogenerated.add(" CLONERS_WITH_ID.put(" + className + ".class, (Cloner<" + className + ">) " + clonerImplClassName + "::deepClone);"); } pw.println("}"); } diff --git a/model/map-jpa/src/main/java/org/keycloak/models/map/storage/jpa/JpaMapKeycloakTransaction.java b/model/map-jpa/src/main/java/org/keycloak/models/map/storage/jpa/JpaMapKeycloakTransaction.java index 032dd8ba1f..a0455a03c2 100644 --- a/model/map-jpa/src/main/java/org/keycloak/models/map/storage/jpa/JpaMapKeycloakTransaction.java +++ b/model/map-jpa/src/main/java/org/keycloak/models/map/storage/jpa/JpaMapKeycloakTransaction.java @@ -85,7 +85,7 @@ public abstract class JpaMapKeycloakTransaction { + private static final Logger logger = Logger.getLogger(JpaRoleMapKeycloakTransaction.class); + @SuppressWarnings("unchecked") public JpaRoleMapKeycloakTransaction(KeycloakSession session, EntityManager em) { super(session, JpaRoleEntity.class, RoleModel.class, em); @@ -61,6 +66,16 @@ public class JpaRoleMapKeycloakTransaction extends JpaMapKeycloakTransaction compositeRoles; + @Override + public void setId(String id) { + if (super.getId() == null) { + super.setId(id); + // As the entity will be used when creating the composite roles, it needs to be persisted before that. + // The ID not being set indicates a new entity that hasn't been persisted yet, and the ID is the minimum field for persisting it. + em.persist(original); + } else { + super.setId(id); + } + } + public JpaMapRoleEntityDelegate(JpaRoleEntity original, EntityManager em) { super(new JpaRoleDelegateProvider(original, em)); + this.original = original; this.em = em; } @@ -64,7 +78,9 @@ public class JpaMapRoleEntityDelegate extends MapRoleEntityDelegate { Query query = em.createNamedQuery("deleteAllChildRolesFromCompositeRole"); query.setParameter("roleId", StringKeyConverter.UUIDKey.INSTANCE.fromString(getId())); query.executeUpdate(); - compositeRoles.forEach(this::addCompositeRole); + if (compositeRoles != null) { + compositeRoles.forEach(this::addCompositeRole); + } this.compositeRoles = compositeRoles; } diff --git a/model/map/src/main/java/org/keycloak/models/map/common/AbstractMapProviderFactory.java b/model/map/src/main/java/org/keycloak/models/map/common/AbstractMapProviderFactory.java index 56fb9385ce..30c95ea4a8 100644 --- a/model/map/src/main/java/org/keycloak/models/map/common/AbstractMapProviderFactory.java +++ b/model/map/src/main/java/org/keycloak/models/map/common/AbstractMapProviderFactory.java @@ -105,7 +105,7 @@ public abstract class AbstractMapProviderFactory getStorage(KeycloakSession session) { + public MapStorage getStorage(KeycloakSession session) { ProviderFactory storageProviderFactory = getProviderFactoryOrComponentFactory(session, storageConfigScope); final MapStorageProvider factory = storageProviderFactory.create(session); session.enlistForClose(factory); diff --git a/model/map/src/main/java/org/keycloak/models/map/datastore/ImportKeycloakSession.java b/model/map/src/main/java/org/keycloak/models/map/datastore/ImportKeycloakSession.java new file mode 100644 index 0000000000..4c890f501b --- /dev/null +++ b/model/map/src/main/java/org/keycloak/models/map/datastore/ImportKeycloakSession.java @@ -0,0 +1,373 @@ +/* + * Copyright 2022 Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.keycloak.models.map.datastore; + +import org.jboss.logging.Logger; +import org.keycloak.Config; +import org.keycloak.component.ComponentModel; +import org.keycloak.models.ClientProvider; +import org.keycloak.models.ClientScopeProvider; +import org.keycloak.models.ClientScopeSpi; +import org.keycloak.models.ClientSpi; +import org.keycloak.models.GroupProvider; +import org.keycloak.models.GroupSpi; +import org.keycloak.models.KeyManager; +import org.keycloak.models.KeycloakContext; +import org.keycloak.models.KeycloakSession; +import org.keycloak.models.KeycloakSessionFactory; +import org.keycloak.models.KeycloakTransactionManager; +import org.keycloak.models.ModelException; +import org.keycloak.models.RealmProvider; +import org.keycloak.models.RealmSpi; +import org.keycloak.models.RoleProvider; +import org.keycloak.models.RoleSpi; +import org.keycloak.models.ThemeManager; +import org.keycloak.models.TokenManager; +import org.keycloak.models.UserCredentialManager; +import org.keycloak.models.UserLoginFailureProvider; +import org.keycloak.models.UserProvider; +import org.keycloak.models.UserSessionProvider; +import org.keycloak.models.UserSpi; +import org.keycloak.models.map.client.MapClientProvider; +import org.keycloak.models.map.client.MapClientProviderFactory; +import org.keycloak.models.map.clientscope.MapClientScopeProvider; +import org.keycloak.models.map.clientscope.MapClientScopeProviderFactory; +import org.keycloak.models.map.common.AbstractEntity; +import org.keycloak.models.map.common.AbstractMapProviderFactory; +import org.keycloak.models.map.group.MapGroupProvider; +import org.keycloak.models.map.group.MapGroupProviderFactory; +import org.keycloak.models.map.realm.MapRealmProvider; +import org.keycloak.models.map.realm.MapRealmProviderFactory; +import org.keycloak.models.map.role.MapRoleProvider; +import org.keycloak.models.map.role.MapRoleProviderFactory; +import org.keycloak.models.map.user.MapUserProvider; +import org.keycloak.models.map.user.MapUserProviderFactory; +import org.keycloak.provider.InvalidationHandler; +import org.keycloak.provider.Provider; +import org.keycloak.provider.Spi; +import org.keycloak.services.clientpolicy.ClientPolicyManager; +import org.keycloak.sessions.AuthenticationSessionProvider; +import org.keycloak.vault.VaultTranscriber; + +import java.lang.reflect.InvocationTargetException; +import java.util.HashSet; +import java.util.Set; +import java.util.function.Function; + +import static org.keycloak.common.util.StackUtil.getShortStackTrace; + +/** + * This implementation of {@link KeycloakSession} wraps an existing session and directs all calls to the datastore provider + * to a separate {@link KeycloakSessionFactory}. + * This allows it to create instantiate different storage providers during import. + * + * @author Alexander Schwartz + */ +public class ImportKeycloakSession implements KeycloakSession { + + private static final Logger LOG = Logger.getLogger(ImportKeycloakSession.class); + + private final KeycloakSessionFactory factory; + private final KeycloakSession session; + private final MapRealmProvider realmProvider; + private final MapClientProvider clientProvider; + private final MapClientScopeProvider clientScopeProvider; + private final MapGroupProvider groupProvider; + private final MapRoleProvider roleProvider; + private final MapUserProvider userProvider; + + private final Set> providerFactories = new HashSet<>(); + + public ImportKeycloakSession(ImportSessionFactoryWrapper factory, KeycloakSession session) { + this.factory = factory; + this.session = session; + realmProvider = createProvider(RealmSpi.class, MapRealmProviderFactory.class); + clientProvider = createProvider(ClientSpi.class, MapClientProviderFactory.class); + clientScopeProvider = createProvider(ClientScopeSpi.class, MapClientScopeProviderFactory.class); + groupProvider = createProvider(GroupSpi.class, MapGroupProviderFactory.class); + roleProvider = createProvider(RoleSpi.class, MapRoleProviderFactory.class); + userProvider = createProvider(UserSpi.class, MapUserProviderFactory.class); + } + + private

P createProvider(Class spi, Class> providerFactoryClass) { + try { + AbstractMapProviderFactory providerFactory = providerFactoryClass.getConstructor().newInstance(); + providerFactory.init(Config.scope(spi.getDeclaredConstructor().newInstance().getName(), providerFactory.getId())); + providerFactories.add(providerFactory); + return providerFactory.create(this); + } catch (InstantiationException | IllegalAccessException | NoSuchMethodException | + InvocationTargetException e) { + throw new RuntimeException(e); + } + } + + @Override + public KeycloakContext getContext() { + return session.getContext(); + } + + @Override + public KeycloakTransactionManager getTransactionManager() { + return session.getTransactionManager(); + } + + @Override + public T getProvider(Class clazz) { + LOG.warnf("Calling getProvider(%s) on the parent session. Revisit this to ensure it doesn't have side effects on the parent session.", clazz.getName(), getShortStackTrace()); + return session.getProvider(clazz); + } + + @Override + public T getProvider(Class clazz, String id) { + LOG.warnf("Calling getProvider(%s, %s) on the parent session. Revisit this to ensure it doesn't have side effects on the parent session.", clazz.getName(), id, getShortStackTrace()); + return session.getProvider(clazz, id); + } + + @Override + public T getComponentProvider(Class clazz, String componentId) { + LOG.warnf("Calling getComponentProvider(%s, %s) on the parent session. Revisit this to ensure it doesn't have side effects on the parent session.", clazz.getName(), componentId, getShortStackTrace()); + return session.getComponentProvider(clazz, componentId); + } + + @Override + public T getComponentProvider(Class clazz, String componentId, Function modelGetter) { + LOG.warnf("Calling getComponentProvider(%s, %s, ...) on the parent session. Revisit this to ensure it doesn't have side effects on the parent session.", clazz.getName(), componentId, getShortStackTrace()); + return session.getComponentProvider(clazz, componentId, modelGetter); + } + + @Override + public T getProvider(Class clazz, ComponentModel componentModel) { + LOG.warnf("Calling getProvider(%s, ...) on the parent session. Revisit this to ensure it doesn't have side effects on the parent session.", clazz.getName(), getShortStackTrace()); + return session.getProvider(clazz, componentModel); + } + + @Override + public Set listProviderIds(Class clazz) { + LOG.warnf("Calling listProviderIds(%s, ...) on the parent session. Revisit this to ensure it doesn't have side effects on the parent session.", clazz.getName(), getShortStackTrace()); + return session.listProviderIds(clazz); + } + + @Override + public Set getAllProviders(Class clazz) { + LOG.warnf("Calling getAllProviders(%s) on the parent session. Revisit this to ensure it doesn't have side effects on the parent session.", clazz.getName(), getShortStackTrace()); + return session.getAllProviders(clazz); + } + + @Override + public Class getProviderClass(String providerClassName) { + LOG.warnf("Calling getProviderClass(%s) on the parent session. Revisit this to ensure it doesn't have side effects on the parent session.", providerClassName, getShortStackTrace()); + return session.getProviderClass(providerClassName); + } + + @Override + public Object getAttribute(String attribute) { + return session.getAttribute(attribute); + } + + @Override + public T getAttribute(String attribute, Class clazz) { + return session.getAttribute(attribute, clazz); + } + + @Override + public T getAttributeOrDefault(String attribute, T defaultValue) { + return session.getAttributeOrDefault(attribute, defaultValue); + } + + @Override + public Object removeAttribute(String attribute) { + return session.removeAttribute(attribute); + } + + @Override + public void setAttribute(String name, Object value) { + session.setAttribute(name, value); + } + + @Override + public void invalidate(InvalidationHandler.InvalidableObjectType type, Object... ids) { + // For now, forward the invalidations only to those providers created specifically for this import session. + providerFactories.stream() + .filter(InvalidationHandler.class::isInstance) + .map(InvalidationHandler.class::cast) + .forEach(ih -> ih.invalidate(this, type, ids)); + } + + @Override + public void enlistForClose(Provider provider) { + session.enlistForClose(provider); + } + + @Override + public KeycloakSessionFactory getKeycloakSessionFactory() { + return factory; + } + + @Override + public RealmProvider realms() { + return realmProvider; + } + + @Override + public ClientProvider clients() { + return clientProvider; + } + + @Override + public ClientScopeProvider clientScopes() { + return clientScopeProvider; + } + + @Override + public GroupProvider groups() { + return groupProvider; + } + + @Override + public RoleProvider roles() { + return roleProvider; + } + + @Override + public UserSessionProvider sessions() { + throw new ModelException("not supported yet"); + } + + @Override + public UserLoginFailureProvider loginFailures() { + throw new ModelException("not supported yet"); + } + + @Override + public AuthenticationSessionProvider authenticationSessions() { + throw new ModelException("not supported yet"); + } + + @Override + public void close() { + session.close(); + } + + @Override + @Deprecated + public UserProvider userCache() { + throw new ModelException("not supported"); + } + + @Override + public UserProvider users() { + return userProvider; + } + + @Override + @Deprecated + public ClientProvider clientStorageManager() { + throw new ModelException("not supported"); + } + + @Override + @Deprecated + public ClientScopeProvider clientScopeStorageManager() { + throw new ModelException("not supported"); + } + + @Override + @Deprecated + public RoleProvider roleStorageManager() { + throw new ModelException("not supported"); + } + + @Override + @Deprecated + public GroupProvider groupStorageManager() { + throw new ModelException("not supported"); + } + + @Override + @Deprecated + public UserProvider userStorageManager() { + throw new ModelException("not supported"); + } + + @Override + @Deprecated + public UserCredentialManager userCredentialManager() { + throw new ModelException("not supported"); + } + + @Override + @Deprecated + public UserProvider userLocalStorage() { + throw new ModelException("not supported"); + } + + @Override + @Deprecated + public RealmProvider realmLocalStorage() { + throw new ModelException("not supported"); + } + + @Override + @Deprecated + public ClientProvider clientLocalStorage() { + throw new ModelException("not supported"); + } + + @Override + @Deprecated + public ClientScopeProvider clientScopeLocalStorage() { + throw new ModelException("not supported"); + } + + @Override + @Deprecated + public GroupProvider groupLocalStorage() { + throw new ModelException("not supported"); + } + + @Override + @Deprecated + public RoleProvider roleLocalStorage() { + throw new ModelException("not supported"); + } + + @Override + public KeyManager keys() { + throw new ModelException("not supported"); + } + + @Override + public ThemeManager theme() { + throw new ModelException("not supported"); + } + + @Override + public TokenManager tokens() { + throw new ModelException("not supported"); + } + + @Override + public VaultTranscriber vault() { + throw new ModelException("not supported"); + } + + @Override + public ClientPolicyManager clientPolicy() { + return session.clientPolicy(); + } +} diff --git a/model/map/src/main/java/org/keycloak/models/map/datastore/ImportSessionFactoryWrapper.java b/model/map/src/main/java/org/keycloak/models/map/datastore/ImportSessionFactoryWrapper.java new file mode 100644 index 0000000000..a1a7afab2e --- /dev/null +++ b/model/map/src/main/java/org/keycloak/models/map/datastore/ImportSessionFactoryWrapper.java @@ -0,0 +1,128 @@ +/* + * Copyright 2022 Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.keycloak.models.map.datastore; + +import org.keycloak.Config; +import org.keycloak.component.ComponentModel; +import org.keycloak.models.KeycloakSession; +import org.keycloak.models.KeycloakSessionFactory; +import org.keycloak.models.map.storage.MapStorageProvider; +import org.keycloak.models.map.storage.MapStorageSpi; +import org.keycloak.models.map.storage.chm.ConcurrentHashMapStorageProviderFactory; +import org.keycloak.provider.Provider; +import org.keycloak.provider.ProviderEvent; +import org.keycloak.provider.ProviderEventListener; +import org.keycloak.provider.ProviderFactory; +import org.keycloak.provider.Spi; + +import java.util.List; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Stream; + +/** + * This wraps an existing KeycloakSessionFactory and redirects all calls to a {@link MapStorageProvider} to + * {@link org.keycloak.models.map.storage.chm.ConcurrentHashMapStorageProvider}. This allows all operations to + * be in-memory. The final contents of the store can then be copied over to the final store once the import is complete. + *

+ * For this to work, the CHM provider needs to be registered as a provider. + * + * @author Alexander Schwartz + */ +public class ImportSessionFactoryWrapper implements KeycloakSessionFactory { + private ConcurrentHashMapStorageProviderFactory concurrentHashMapStorageProviderFactory; + + @Override + public KeycloakSession create() { + concurrentHashMapStorageProviderFactory = new ConcurrentHashMapStorageProviderFactory(); + concurrentHashMapStorageProviderFactory.init(Config.scope(MapStorageSpi.NAME, concurrentHashMapStorageProviderFactory.getId())); + return new ImportKeycloakSession(this, keycloakSessionFactory.create()); + } + + @Override + public Set getSpis() { + return keycloakSessionFactory.getSpis(); + } + + @Override + public Spi getSpi(Class providerClass) { + return keycloakSessionFactory.getSpi(providerClass); + } + + @Override + public ProviderFactory getProviderFactory(Class clazz) { + if (clazz == MapStorageProvider.class) { + return keycloakSessionFactory.getProviderFactory(clazz, ConcurrentHashMapStorageProviderFactory.PROVIDER_ID); + } + return keycloakSessionFactory.getProviderFactory(clazz); + } + + @Override + public ProviderFactory getProviderFactory(Class clazz, String id) { + if (clazz == MapStorageProvider.class) { + return (ProviderFactory) concurrentHashMapStorageProviderFactory; + } + return keycloakSessionFactory.getProviderFactory(clazz, id); + } + + @Override + public ProviderFactory getProviderFactory(Class clazz, String realmId, String componentId, Function modelGetter) { + return keycloakSessionFactory.getProviderFactory(clazz, realmId, componentId, modelGetter); + } + + @Override + public Stream getProviderFactoriesStream(Class clazz) { + return keycloakSessionFactory.getProviderFactoriesStream(clazz); + } + + @Override + public long getServerStartupTimestamp() { + return keycloakSessionFactory.getServerStartupTimestamp(); + } + + @Override + public void close() { + keycloakSessionFactory.close(); + } + + @Override + public void invalidate(KeycloakSession session, InvalidableObjectType type, Object... params) { + keycloakSessionFactory.invalidate(session, type, params); + } + + @Override + public void register(ProviderEventListener listener) { + keycloakSessionFactory.register(listener); + } + + @Override + public void unregister(ProviderEventListener listener) { + keycloakSessionFactory.unregister(listener); + } + + @Override + public void publish(ProviderEvent event) { + keycloakSessionFactory.publish(event); + } + + private final KeycloakSessionFactory keycloakSessionFactory; + + public ImportSessionFactoryWrapper(KeycloakSessionFactory keycloakSessionFactory) { + this.keycloakSessionFactory = keycloakSessionFactory; + } +} diff --git a/model/map/src/main/java/org/keycloak/models/map/datastore/MapExportImportManager.java b/model/map/src/main/java/org/keycloak/models/map/datastore/MapExportImportManager.java index b159526258..5e1c0ee427 100644 --- a/model/map/src/main/java/org/keycloak/models/map/datastore/MapExportImportManager.java +++ b/model/map/src/main/java/org/keycloak/models/map/datastore/MapExportImportManager.java @@ -18,9 +18,15 @@ package org.keycloak.models.map.datastore; import org.jboss.logging.Logger; +import org.keycloak.Config; import org.keycloak.common.enums.SslRequired; import org.keycloak.common.util.MultivaluedHashMap; import org.keycloak.component.ComponentModel; +import org.keycloak.models.AdminRoles; +import org.keycloak.models.ClientProvider; +import org.keycloak.models.ClientScopeProvider; +import org.keycloak.models.GroupProvider; +import org.keycloak.models.ImpersonationConstants; import org.keycloak.models.ModelException; import org.keycloak.exportimport.ExportAdapter; import org.keycloak.exportimport.ExportOptions; @@ -43,18 +49,29 @@ import org.keycloak.models.OTPPolicy; import org.keycloak.models.ParConfig; import org.keycloak.models.PasswordPolicy; import org.keycloak.models.RealmModel; +import org.keycloak.models.RealmProvider; import org.keycloak.models.RequiredActionProviderModel; import org.keycloak.models.RoleModel; +import org.keycloak.models.RoleProvider; import org.keycloak.models.ScopeContainerModel; import org.keycloak.models.UserConsentModel; import org.keycloak.models.UserModel; +import org.keycloak.models.UserProvider; import org.keycloak.models.WebAuthnPolicy; +import org.keycloak.models.map.common.AbstractEntity; +import org.keycloak.models.map.common.AbstractMapProviderFactory; +import org.keycloak.models.map.realm.MapRealmEntity; +import org.keycloak.models.map.storage.MapKeycloakTransaction; +import org.keycloak.models.map.storage.ModelCriteriaBuilder; +import org.keycloak.models.map.storage.criteria.DefaultModelCriteria; import org.keycloak.models.utils.DefaultAuthenticationFlows; import org.keycloak.models.utils.DefaultKeyProviders; import org.keycloak.models.utils.DefaultRequiredActions; import org.keycloak.models.utils.KeycloakModelUtils; import org.keycloak.models.utils.RepresentationToModel; import org.keycloak.protocol.oidc.OIDCConfigAttributes; +import org.keycloak.provider.Provider; +import org.keycloak.provider.ProviderFactory; import org.keycloak.representations.idm.ApplicationRepresentation; import org.keycloak.representations.idm.AuthenticationExecutionExportRepresentation; import org.keycloak.representations.idm.AuthenticationFlowRepresentation; @@ -76,12 +93,14 @@ import org.keycloak.representations.idm.RequiredActionProviderRepresentation; import org.keycloak.representations.idm.RoleRepresentation; import org.keycloak.representations.idm.ScopeMappingRepresentation; import org.keycloak.representations.idm.UserConsentRepresentation; -import org.keycloak.representations.idm.UserFederationMapperRepresentation; import org.keycloak.representations.idm.UserRepresentation; import org.keycloak.storage.ExportImportManager; import org.keycloak.storage.ImportRealmFromRepresentation; +import org.keycloak.storage.SearchableModelField; +import org.keycloak.storage.SetDefaultsForNewRealm; import org.keycloak.userprofile.UserProfileProvider; import org.keycloak.util.JsonSerialization; +import org.keycloak.utils.ReservedCharValidator; import org.keycloak.validation.ValidationUtil; import java.io.IOException; @@ -93,10 +112,14 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import static org.keycloak.models.map.storage.QueryParameters.withCriteria; +import static org.keycloak.models.map.storage.criteria.DefaultModelCriteria.criteria; import static org.keycloak.models.utils.RepresentationToModel.createCredentials; import static org.keycloak.models.utils.RepresentationToModel.createFederatedIdentities; import static org.keycloak.models.utils.RepresentationToModel.createGroups; @@ -119,8 +142,21 @@ public class MapExportImportManager implements ExportImportManager { private final KeycloakSession session; private static final Logger logger = Logger.getLogger(MapExportImportManager.class); + /** + * Use the old import via the logical layer vs. the new method importing to CHM first and then copying over + * This is a temporary to test the functionality with the old representations until the new file store arrives in main, + * and will then be removed. + */ + private final boolean useNewImportMethod; + public MapExportImportManager(KeycloakSession session) { this.session = session; + useNewImportMethod = Boolean.parseBoolean(System.getProperty(MapExportImportManager.class.getName(), "false")); + } + + public MapExportImportManager(KeycloakSession session, boolean useNewImportMethod) { + this.session = session; + this.useNewImportMethod = useNewImportMethod; } @Override @@ -403,11 +439,32 @@ public class MapExportImportManager implements ExportImportManager { } } - if (newRealm.getComponentsStream(newRealm.getId(), KeyProvider.class.getName()).count() == 0) { + if (!useNewImportMethod) { + if (newRealm.getComponentsStream(newRealm.getId(), KeyProvider.class.getName()).count() == 0) { + if (rep.getPrivateKey() != null) { + DefaultKeyProviders.createProviders(newRealm, rep.getPrivateKey(), rep.getCertificate()); + } else { + DefaultKeyProviders.createProviders(newRealm); + } + } + } else { if (rep.getPrivateKey() != null) { - DefaultKeyProviders.createProviders(newRealm, rep.getPrivateKey(), rep.getCertificate()); - } else { - DefaultKeyProviders.createProviders(newRealm); + + ComponentModel rsa = new ComponentModel(); + rsa.setName("rsa"); + rsa.setParentId(newRealm.getId()); + rsa.setProviderId("rsa"); + rsa.setProviderType(KeyProvider.class.getName()); + + MultivaluedHashMap config = new MultivaluedHashMap<>(); + config.putSingle("priority", "100"); + config.putSingle("privateKey", rep.getPrivateKey()); + if (rep.getCertificate() != null) { + config.putSingle("certificate", rep.getCertificate()); + } + rsa.setConfig(config); + + newRealm.addComponentModel(rsa); } } } @@ -437,11 +494,230 @@ public class MapExportImportManager implements ExportImportManager { } logger.debugv("importRealm: {0}", rep.getRealm()); - /* The import for the JSON representation might be called from the Admin UI, where it will be empty except for - the realm name and if the realm is enabled. For that scenario, it would need to create all missing elements, - which is done by firing an event to call the existing implementation in the RealmManager. */ + if (!useNewImportMethod) { + /* The import for the JSON representation might be called from the Admin UI, where it will be empty except for + the realm name and if the realm is enabled. For that scenario, it would need to create all missing elements, + which is done by firing an event to call the existing implementation in the RealmManager. */ + return ImportRealmFromRepresentation.fire(session, rep); + } else { + /* This makes use of the representation to mimic the future setup: Some kind of import into a ConcurrentHashMap in-memory and then copying + that over to the real store. This is the basis for future file store import. Results are different + when importing, for example, an empty list of roles vs a non-existing list of roles, and possibility in other ways. + Importing from a classic representation will eventually be removed and replaced when the new file store arrived. */ + return importToChmAndThenCopyOver(rep); + } + } + + private RealmModel importToChmAndThenCopyOver(RealmRepresentation rep) { + String id = rep.getId(); + if (id == null || id.trim().isEmpty()) { + id = KeycloakModelUtils.generateId(); + } else { + ReservedCharValidator.validate(id); + } + + ReservedCharValidator.validate(rep.getRealm()); + + RealmModel realm; + RealmModel currentRealm = session.getContext().getRealm(); + + try { + + String _id = id; + KeycloakModelUtils.runJobInTransaction(new ImportSessionFactoryWrapper(session.getKeycloakSessionFactory()), chmSession -> { + // import the representation + fillRealm(chmSession, _id, rep); + + // copy over the realm from in-memory to the real + copyRealm(_id, chmSession); + copyEntities(_id, chmSession, ClientProvider.class, ClientModel.class, ClientModel.SearchableFields.REALM_ID); + copyEntities(_id, chmSession, ClientScopeProvider.class, ClientScopeModel.class, ClientScopeModel.SearchableFields.REALM_ID); + copyEntities(_id, chmSession, GroupProvider.class, GroupModel.class, GroupModel.SearchableFields.REALM_ID); + copyEntities(_id, chmSession, UserProvider.class, UserModel.class, UserModel.SearchableFields.REALM_ID); + copyEntities(_id, chmSession, RoleProvider.class, RoleModel.class, RoleModel.SearchableFields.REALM_ID); + + // clear the CHM store + chmSession.getTransactionManager().setRollbackOnly(); + }); + + realm = session.realms().getRealm(id); + session.getContext().setRealm(realm); + setupMasterAdminManagement(realm); + ImpersonationConstants.setupImpersonationService(session, realm); + fireRealmPostCreate(realm); + } finally { + session.getContext().setRealm(currentRealm); + } + + return realm; + } + + private void copyRealm(String realmId, KeycloakSession sessionChm) { + MapRealmEntity realmEntityChm = (MapRealmEntity) getTransaction(sessionChm, RealmProvider.class).read(realmId); + getTransaction(session, RealmProvider.class).create(realmEntityChm); + } + + private static

MapKeycloakTransaction getTransaction(KeycloakSession session, Class

provider) { + ProviderFactory

factoryChm = session.getKeycloakSessionFactory().getProviderFactory(provider); + return ((AbstractMapProviderFactory) factoryChm).getStorage(session).createTransaction(session); + } + + private

void copyEntities(String realmId, KeycloakSession sessionChm, Class

provider, Class model, SearchableModelField field) { + MapKeycloakTransaction txChm = getTransaction(sessionChm, provider); + MapKeycloakTransaction txOrig = getTransaction(session, provider); + + DefaultModelCriteria mcb = criteria(); + mcb = mcb.compare(field, ModelCriteriaBuilder.Operator.EQ, realmId); + + txChm.read(withCriteria(mcb)).forEach(txOrig::create); + } + + private static void fillRealm(KeycloakSession session, String id, RealmRepresentation rep) { + RealmModel realm = session.realms().createRealm(id, rep.getRealm()); + session.getContext().setRealm(realm); + SetDefaultsForNewRealm.fire(session, realm); + MapExportImportManager mapExportImportManager = new MapExportImportManager(session); + mapExportImportManager.clearDefaultsThatConflictWithRepresentation(rep, realm); + mapExportImportManager.importRealm(rep, realm, false); + } + + private void clearDefaultsThatConflictWithRepresentation(RealmRepresentation rep, RealmModel newRealm) { + if (rep.getDefaultRole() != null) { + if (newRealm.getDefaultRole() != null) { + newRealm.removeRole(newRealm.getDefaultRole()); + // set the new role here already as the legacy code expects it this way + newRealm.setDefaultRole(RepresentationToModel.createRole(newRealm, rep.getDefaultRole())); + } + } + + if (rep.getRequiredActions() != null) { + for (RequiredActionProviderRepresentation action : rep.getRequiredActions()) { + RequiredActionProviderModel requiredActionProviderByAlias = newRealm.getRequiredActionProviderByAlias(action.getAlias()); + if (requiredActionProviderByAlias != null) { + newRealm.removeRequiredActionProvider(requiredActionProviderByAlias); + } + } + } + + if (rep.getRoles() != null) { + for (RoleRepresentation representation : rep.getRoles().getRealm()) { + RoleModel role = newRealm.getRole(representation.getName()); + if (role != null && (newRealm.getDefaultRole() == null || newRealm.getDefaultRole() != null && !Objects.equals(role.getId(), newRealm.getDefaultRole().getId()))) { + newRealm.removeRole(role); + } + } + } + + if (rep.getPrivateKey() != null) { + newRealm.getComponentsStream(newRealm.getId(), KeyProvider.class.getName()) + .filter(component -> Objects.equals(component.getProviderId(), "rsa-generated") || Objects.equals(component.getProviderId(), "rsa-enc-generated")) + .collect(Collectors.toList()).forEach(newRealm::removeComponent); + // will later create the "rsa" provider + } + + if (rep.getClients() != null) { + for (ClientRepresentation resourceRep : rep.getClients()) { + ClientModel clientByClientId = newRealm.getClientByClientId(resourceRep.getClientId()); + if (clientByClientId != null) { + newRealm.removeClient(clientByClientId.getId()); + } + } + } + + if (rep.getClientScopes() != null) { + for (ClientScopeRepresentation resourceRep : rep.getClientScopes()) { + Optional existingClientScope = newRealm.getClientScopesStream().filter(clientScopeModel -> clientScopeModel.getName().equals(resourceRep.getName())).findFirst(); + if (existingClientScope.isPresent()) { + newRealm.removeClientScope(existingClientScope.get().getId()); + } + } + } + + if (rep.getComponents() != null) { + clearExistingComponents(newRealm, rep.getComponents()); + } + } + + protected static void clearExistingComponents(RealmModel newRealm, MultivaluedHashMap components) { + for (Map.Entry> entry : components.entrySet()) { + String providerType = entry.getKey(); + for (ComponentExportRepresentation compRep : entry.getValue()) { + newRealm.getComponentsStream(newRealm.getId(), providerType) + .filter(component -> Objects.equals(component.getProviderId(), compRep.getProviderId())).findAny().ifPresent(newRealm::removeComponent); + if (compRep.getSubComponents() != null) { + clearExistingComponents(newRealm, compRep.getSubComponents()); + } + } + } + } + + public void setupMasterAdminManagement(RealmModel realm) { + // Need to refresh masterApp for current realm + String adminRealmName = Config.getAdminRealm(); + RealmModel adminRealm = session.realms().getRealmByName(adminRealmName); + ClientModel masterApp = adminRealm.getClientByClientId(KeycloakModelUtils.getMasterRealmAdminApplicationClientId(realm.getName())); + if (masterApp == null) { + createMasterAdminManagement(realm); + return; + } + realm.setMasterAdminClient(masterApp); + } + + private void createMasterAdminManagement(RealmModel realm) { + RealmModel adminRealm; + RoleModel adminRole; + + if (realm.getName().equals(Config.getAdminRealm())) { + adminRealm = realm; + + adminRole = realm.addRole(AdminRoles.ADMIN); + + RoleModel createRealmRole = realm.addRole(AdminRoles.CREATE_REALM); + adminRole.addCompositeRole(createRealmRole); + createRealmRole.setDescription("${role_" + AdminRoles.CREATE_REALM + "}"); + } else { + adminRealm = session.realms().getRealmByName(Config.getAdminRealm()); + adminRole = adminRealm.getRole(AdminRoles.ADMIN); + } + adminRole.setDescription("${role_" + AdminRoles.ADMIN + "}"); + + ClientModel realmAdminApp = KeycloakModelUtils.createManagementClient(adminRealm, KeycloakModelUtils.getMasterRealmAdminApplicationClientId(realm.getName())); + // No localized name for now + realmAdminApp.setName(realm.getName() + " Realm"); + realm.setMasterAdminClient(realmAdminApp); + + for (String r : AdminRoles.ALL_REALM_ROLES) { + RoleModel role = realmAdminApp.addRole(r); + role.setDescription("${role_" + r + "}"); + adminRole.addCompositeRole(role); + } + addQueryCompositeRoles(realmAdminApp); + } + + public void addQueryCompositeRoles(ClientModel realmAccess) { + RoleModel queryClients = realmAccess.getRole(AdminRoles.QUERY_CLIENTS); + RoleModel queryUsers = realmAccess.getRole(AdminRoles.QUERY_USERS); + RoleModel queryGroups = realmAccess.getRole(AdminRoles.QUERY_GROUPS); + + RoleModel viewClients = realmAccess.getRole(AdminRoles.VIEW_CLIENTS); + viewClients.addCompositeRole(queryClients); + RoleModel viewUsers = realmAccess.getRole(AdminRoles.VIEW_USERS); + viewUsers.addCompositeRole(queryUsers); + viewUsers.addCompositeRole(queryGroups); + } + + private void fireRealmPostCreate(RealmModel realm) { + session.getKeycloakSessionFactory().publish(new RealmModel.RealmPostCreateEvent() { + @Override + public RealmModel getCreatedRealm() { + return realm; + } + @Override + public KeycloakSession getKeycloakSession() { + return session; + } + }); - return ImportRealmFromRepresentation.fire(session, rep); } private static void convertDeprecatedDefaultRoles(RealmRepresentation rep, RealmModel newRealm) { @@ -1005,20 +1281,6 @@ public class MapExportImportManager implements ExportImportManager { } - public static ComponentModel convertFedMapperToComponent(RealmModel realm, ComponentModel parent, UserFederationMapperRepresentation rep, String newMapperType) { - ComponentModel mapper = new ComponentModel(); - mapper.setId(rep.getId()); - mapper.setName(rep.getName()); - mapper.setProviderId(rep.getFederationMapperType()); - mapper.setProviderType(newMapperType); - mapper.setParentId(parent.getId()); - if (rep.getConfig() != null) { - for (Map.Entry entry : rep.getConfig().entrySet()) { - mapper.getConfig().putSingle(entry.getKey(), entry.getValue()); - } - } - return mapper; - } protected static void importComponents(RealmModel newRealm, MultivaluedHashMap components, String parentId) { for (Map.Entry> entry : components.entrySet()) { diff --git a/server-spi-private/src/main/java/org/keycloak/storage/SetDefaultsForNewRealm.java b/server-spi-private/src/main/java/org/keycloak/storage/SetDefaultsForNewRealm.java new file mode 100644 index 0000000000..93fc2b15dd --- /dev/null +++ b/server-spi-private/src/main/java/org/keycloak/storage/SetDefaultsForNewRealm.java @@ -0,0 +1,51 @@ +/* + * Copyright 2022 Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.keycloak.storage; + +import org.keycloak.models.KeycloakSession; +import org.keycloak.models.RealmModel; +import org.keycloak.provider.ProviderEvent; + +/** + * Event to trigger that will add defaults for a realm after it has been imported. + * + * @author Alexander Schwartz + */ +public class SetDefaultsForNewRealm implements ProviderEvent { + private final KeycloakSession session; + private final RealmModel realmModel; + + public SetDefaultsForNewRealm(KeycloakSession session, RealmModel realmModel) { + this.session = session; + this.realmModel = realmModel; + } + + public static void fire(KeycloakSession session, RealmModel realm) { + SetDefaultsForNewRealm event = new SetDefaultsForNewRealm(session, realm); + session.getKeycloakSessionFactory().publish(event); + } + + public KeycloakSession getSession() { + return session; + } + + public RealmModel getRealmModel() { + return realmModel; + } +} + diff --git a/services/src/main/java/org/keycloak/utils/ReservedCharValidator.java b/server-spi-private/src/main/java/org/keycloak/utils/ReservedCharValidator.java similarity index 100% rename from services/src/main/java/org/keycloak/utils/ReservedCharValidator.java rename to server-spi-private/src/main/java/org/keycloak/utils/ReservedCharValidator.java diff --git a/services/src/main/java/org/keycloak/services/managers/RealmManager.java b/services/src/main/java/org/keycloak/services/managers/RealmManager.java index 1ff88d4a2f..7a46015293 100755 --- a/services/src/main/java/org/keycloak/services/managers/RealmManager.java +++ b/services/src/main/java/org/keycloak/services/managers/RealmManager.java @@ -35,6 +35,7 @@ import org.keycloak.models.UserModel; import org.keycloak.models.UserSessionProvider; import org.keycloak.models.utils.DefaultAuthenticationFlows; import org.keycloak.models.utils.DefaultClientScopes; +import org.keycloak.models.utils.DefaultKeyProviders; import org.keycloak.models.utils.DefaultRequiredActions; import org.keycloak.models.utils.KeycloakModelUtils; import org.keycloak.models.utils.RepresentationToModel; @@ -778,4 +779,23 @@ public class RealmManager { } } + public void setDefaultsForNewRealm(RealmModel realm) { + // setup defaults + setupRealmDefaults(realm); + KeycloakModelUtils.setupDefaultRole(realm, Constants.DEFAULT_ROLES_ROLE_PREFIX + "-" + realm.getName().toLowerCase()); + setupRealmAdminManagement(realm); + setupAccountManagement(realm); + setupBrokerService(realm); + setupAdminConsole(realm); + setupAdminConsoleLocaleMapper(realm); + setupAdminCli(realm); + setupAuthenticationFlows(realm); + setupRequiredActions(realm); + setupOfflineTokens(realm, null); + createDefaultClientScopes(realm); + setupAuthorizationServices(realm); + setupClientRegistrations(realm); + session.clientPolicy().setupClientPoliciesOnCreatedRealm(realm); + DefaultKeyProviders.createProviders(realm); + } } diff --git a/services/src/main/java/org/keycloak/services/managers/RealmManagerProviderFactory.java b/services/src/main/java/org/keycloak/services/managers/RealmManagerProviderFactory.java index e71b436d83..5de00fb388 100644 --- a/services/src/main/java/org/keycloak/services/managers/RealmManagerProviderFactory.java +++ b/services/src/main/java/org/keycloak/services/managers/RealmManagerProviderFactory.java @@ -25,6 +25,7 @@ import org.keycloak.models.RealmModel; import org.keycloak.provider.Provider; import org.keycloak.provider.ProviderFactory; import org.keycloak.storage.ImportRealmFromRepresentation; +import org.keycloak.storage.SetDefaultsForNewRealm; /** * Provider to listen for {@link org.keycloak.storage.ImportRealmFromRepresentation} events. @@ -50,6 +51,9 @@ public class RealmManagerProviderFactory implements ProviderFactory