Modify RealmAdminResource.partialImport to work with InputStream

Rework existing PartialImportManager to not interfere with transaction handling, and bundle everything related to AdminEventBuild and JAX-RS Repsonses inside the Resource.

Closes #13611
This commit is contained in:
Alexander Schwartz 2022-11-23 14:30:55 +01:00 committed by Hynek Mlnařík
parent dd03137ea7
commit fd152e8a3e
16 changed files with 240 additions and 84 deletions

View file

@ -59,6 +59,7 @@ import org.keycloak.models.utils.DefaultKeyProviders;
import org.keycloak.models.utils.DefaultRequiredActions;
import org.keycloak.models.utils.KeycloakModelUtils;
import org.keycloak.models.utils.RepresentationToModel;
import org.keycloak.partialimport.PartialImportResults;
import org.keycloak.protocol.oidc.OIDCConfigAttributes;
import org.keycloak.representations.idm.ApplicationRepresentation;
import org.keycloak.representations.idm.AuthenticationExecutionExportRepresentation;
@ -76,6 +77,7 @@ import org.keycloak.representations.idm.GroupRepresentation;
import org.keycloak.representations.idm.IdentityProviderMapperRepresentation;
import org.keycloak.representations.idm.IdentityProviderRepresentation;
import org.keycloak.representations.idm.OAuthClientRepresentation;
import org.keycloak.representations.idm.PartialImportRepresentation;
import org.keycloak.representations.idm.ProtocolMapperRepresentation;
import org.keycloak.representations.idm.RealmRepresentation;
import org.keycloak.representations.idm.RequiredActionProviderRepresentation;
@ -87,7 +89,8 @@ import org.keycloak.representations.idm.UserFederationMapperRepresentation;
import org.keycloak.representations.idm.UserFederationProviderRepresentation;
import org.keycloak.representations.idm.UserRepresentation;
import org.keycloak.storage.ExportImportManager;
import org.keycloak.storage.ImportRealmFromRepresentation;
import org.keycloak.storage.ImportRealmFromRepresentationEvent;
import org.keycloak.storage.PartialImportRealmFromRepresentationEvent;
import org.keycloak.storage.UserStoragePrivateUtil;
import org.keycloak.storage.UserStorageProvider;
import org.keycloak.storage.UserStorageProviderModel;
@ -152,7 +155,7 @@ public class LegacyExportImportManager implements ExportImportManager {
throw new ModelException("unable to read contents from stream", e);
}
logger.debugv("importRealm: {0}", rep.getRealm());
return ImportRealmFromRepresentation.fire(session, rep);
return ImportRealmFromRepresentationEvent.fire(session, rep);
}
@Override
@ -452,6 +455,17 @@ public class LegacyExportImportManager implements ExportImportManager {
}
}
@Override
public PartialImportResults partialImportRealm(RealmModel realm, InputStream requestBody) {
PartialImportRepresentation rep;
try {
rep = JsonSerialization.readValue(requestBody, PartialImportRepresentation.class);
} catch (IOException e) {
throw new ModelException("unable to read contents from stream", e);
}
return PartialImportRealmFromRepresentationEvent.fire(session, rep, realm);
}
private static RoleModel getOrAddRealmRole(RealmModel realm, String name) {
RoleModel role = realm.getRole(name);
if (role == null) {

View file

@ -69,6 +69,7 @@ import org.keycloak.models.utils.DefaultKeyProviders;
import org.keycloak.models.utils.DefaultRequiredActions;
import org.keycloak.models.utils.KeycloakModelUtils;
import org.keycloak.models.utils.RepresentationToModel;
import org.keycloak.partialimport.PartialImportResults;
import org.keycloak.protocol.oidc.OIDCConfigAttributes;
import org.keycloak.provider.Provider;
import org.keycloak.provider.ProviderFactory;
@ -87,6 +88,7 @@ import org.keycloak.representations.idm.GroupRepresentation;
import org.keycloak.representations.idm.IdentityProviderMapperRepresentation;
import org.keycloak.representations.idm.IdentityProviderRepresentation;
import org.keycloak.representations.idm.OAuthClientRepresentation;
import org.keycloak.representations.idm.PartialImportRepresentation;
import org.keycloak.representations.idm.ProtocolMapperRepresentation;
import org.keycloak.representations.idm.RealmRepresentation;
import org.keycloak.representations.idm.RequiredActionProviderRepresentation;
@ -95,7 +97,8 @@ import org.keycloak.representations.idm.ScopeMappingRepresentation;
import org.keycloak.representations.idm.UserConsentRepresentation;
import org.keycloak.representations.idm.UserRepresentation;
import org.keycloak.storage.ExportImportManager;
import org.keycloak.storage.ImportRealmFromRepresentation;
import org.keycloak.storage.ImportRealmFromRepresentationEvent;
import org.keycloak.storage.PartialImportRealmFromRepresentationEvent;
import org.keycloak.storage.SearchableModelField;
import org.keycloak.storage.SetDefaultsForNewRealm;
import org.keycloak.userprofile.UserProfileProvider;
@ -498,7 +501,7 @@ public class MapExportImportManager implements ExportImportManager {
/* The import for the JSON representation might be called from the Admin UI, where it will be empty except for
the realm name and if the realm is enabled. For that scenario, it would need to create all missing elements,
which is done by firing an event to call the existing implementation in the RealmManager. */
return ImportRealmFromRepresentation.fire(session, rep);
return ImportRealmFromRepresentationEvent.fire(session, rep);
} else {
/* This makes use of the representation to mimic the future setup: Some kind of import into a ConcurrentHashMap in-memory and then copying
that over to the real store. This is the basis for future file store import. Results are different
@ -508,6 +511,26 @@ public class MapExportImportManager implements ExportImportManager {
}
}
@Override
public PartialImportResults partialImportRealm(RealmModel realm, InputStream requestBody) {
/* A future implementation that would differentiate between the old JSON representations and the new file store
might want to add the file name or the media type as a method parameter to switch between different implementations. */
PartialImportRepresentation rep;
try {
rep = JsonSerialization.readValue(requestBody, PartialImportRepresentation.class);
} catch (IOException e) {
throw new ModelException("unable to read contents from stream", e);
}
/* The import for the legacy JSON representation might be called from the Admin UI, and it allows for several options as part
* of the representation. Therefore, direct this to the service layer with a (temporary) event so that the logic isn't duplicated
* between legacy and map store.
*/
return PartialImportRealmFromRepresentationEvent.fire(session, rep, realm);
}
private RealmModel importToChmAndThenCopyOver(RealmRepresentation rep) {
String id = rep.getId();
if (id == null || id.trim().isEmpty()) {

View file

@ -248,16 +248,24 @@ public final class KeycloakModelUtils {
/**
* Wrap given runnable job into KeycloakTransaction.
*
* @param factory
* @param task
*/
public static void runJobInTransaction(KeycloakSessionFactory factory, KeycloakSessionTask task) {
runJobInTransactionWithResult(factory, session -> {
task.run(session);
return null;
});
}
/**
* Wrap a given callable job into a KeycloakTransaction.
*/
public static <V> V runJobInTransactionWithResult(KeycloakSessionFactory factory, final KeycloakSessionTaskWithResult<V> callable) {
KeycloakSession session = factory.create();
KeycloakTransaction tx = session.getTransactionManager();
V result;
try {
tx.begin();
task.run(session);
result = callable.run(session);
if (tx.isActive()) {
if (tx.getRollbackOnly()) {
@ -274,6 +282,7 @@ public final class KeycloakModelUtils {
} finally {
session.close();
}
return result;
}
/**

View file

@ -21,6 +21,7 @@ import org.keycloak.exportimport.ExportAdapter;
import org.keycloak.exportimport.ExportOptions;
import org.keycloak.models.RealmModel;
import org.keycloak.models.UserModel;
import org.keycloak.partialimport.PartialImportResults;
import org.keycloak.representations.idm.RealmRepresentation;
import org.keycloak.representations.idm.UserRepresentation;
@ -34,6 +35,8 @@ import java.io.InputStream;
public interface ExportImportManager {
void importRealm(RealmRepresentation rep, RealmModel newRealm, boolean skipUserDependent);
PartialImportResults partialImportRealm(RealmModel realm, InputStream requestBody);
void updateRealm(RealmRepresentation rep, RealmModel realm);
UserModel createUser(RealmModel realm, UserRepresentation userRep);

View file

@ -34,19 +34,19 @@ import org.keycloak.representations.idm.RealmRepresentation;
* @author Alexander Schwartz
*/
@Deprecated
public class ImportRealmFromRepresentation implements ProviderEvent {
public class ImportRealmFromRepresentationEvent implements ProviderEvent {
private final KeycloakSession session;
private final RealmRepresentation realmRepresentation;
private RealmModel realmModel;
public ImportRealmFromRepresentation(KeycloakSession session, RealmRepresentation realmRepresentation) {
public ImportRealmFromRepresentationEvent(KeycloakSession session, RealmRepresentation realmRepresentation) {
this.session = session;
this.realmRepresentation = realmRepresentation;
}
public static RealmModel fire(KeycloakSession session, RealmRepresentation rep) {
ImportRealmFromRepresentation event = new ImportRealmFromRepresentation(session, rep);
ImportRealmFromRepresentationEvent event = new ImportRealmFromRepresentationEvent(session, rep);
session.getKeycloakSessionFactory().publish(event);
return event.getRealmModel();
}

View file

@ -0,0 +1,77 @@
/*
* Copyright 2022 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.storage;
import org.keycloak.models.KeycloakSession;
import org.keycloak.models.RealmModel;
import org.keycloak.partialimport.PartialImportResults;
import org.keycloak.provider.ProviderEvent;
import org.keycloak.representations.idm.PartialImportRepresentation;
/**
* Event to trigger that will complete the import for a given realm representation.
* <p />
* This event was created as the import of a JSON via the UI/REST API can be called using a JSON representation that contains
* only the name of the realm and if it is enabled.
* <p />
* In the future, this might not be needed if this is done when the legacy store migration is complete and the functionality
* is bundled within the map storage.
*
* @author Alexander Schwartz
*/
@Deprecated
public class PartialImportRealmFromRepresentationEvent implements ProviderEvent {
private final KeycloakSession session;
private final PartialImportRepresentation rep;
private final RealmModel realm;
private PartialImportResults partialImportResults;
public PartialImportRealmFromRepresentationEvent(KeycloakSession session, PartialImportRepresentation rep, RealmModel realm) {
this.session = session;
this.rep = rep;
this.realm = realm;
}
public static PartialImportResults fire(KeycloakSession session, PartialImportRepresentation rep, RealmModel realm) {
PartialImportRealmFromRepresentationEvent event = new PartialImportRealmFromRepresentationEvent(session, rep, realm);
session.getKeycloakSessionFactory().publish(event);
return event.getPartialImportResults();
}
public KeycloakSession getSession() {
return session;
}
public PartialImportRepresentation getRep() {
return rep;
}
public void setPartialImportResults(PartialImportResults partialImportResults) {
this.partialImportResults = partialImportResults;
}
public PartialImportResults getPartialImportResults() {
return partialImportResults;
}
public RealmModel getRealm() {
return realm;
}
}

View file

@ -25,7 +25,7 @@ import javax.ws.rs.core.Response;
*
* @author Stan Silvert ssilvert@redhat.com (C) 2016 Red Hat Inc.
*/
public class ErrorResponseException extends Exception {
public class ErrorResponseException extends RuntimeException {
private final Response response;
public ErrorResponseException(Response response) {

View file

@ -17,16 +17,10 @@
package org.keycloak.partialimport;
import org.keycloak.events.admin.OperationType;
import org.keycloak.models.KeycloakSession;
import org.keycloak.models.ModelDuplicateException;
import org.keycloak.models.ModelException;
import org.keycloak.models.RealmModel;
import org.keycloak.representations.idm.PartialImportRepresentation;
import org.keycloak.services.ErrorResponse;
import org.keycloak.services.resources.admin.AdminEventBuilder;
import javax.ws.rs.core.Response;
import java.util.ArrayList;
import java.util.List;
@ -41,14 +35,12 @@ public class PartialImportManager {
private final PartialImportRepresentation rep;
private final KeycloakSession session;
private final RealmModel realm;
private final AdminEventBuilder adminEvent;
public PartialImportManager(PartialImportRepresentation rep, KeycloakSession session,
RealmModel realm, AdminEventBuilder adminEvent) {
RealmModel realm) {
this.rep = rep;
this.session = session;
this.realm = realm;
this.adminEvent = adminEvent;
// Do not change the order of these!!!
partialImports.add(new ClientsPartialImport());
@ -59,55 +51,19 @@ public class PartialImportManager {
partialImports.add(new UsersPartialImport());
}
public Response saveResources() {
try {
public PartialImportResults saveResources() throws ErrorResponseException {
PartialImportResults results = new PartialImportResults();
PartialImportResults results = new PartialImportResults();
for (PartialImport partialImport : partialImports) {
partialImport.prepare(rep, realm, session);
}
for (PartialImport partialImport : partialImports) {
partialImport.removeOverwrites(realm, session);
results.addAllResults(partialImport.doImport(rep, realm, session));
}
for (PartialImportResult result : results.getResults()) {
switch (result.getAction()) {
case ADDED : fireCreatedEvent(result); break;
case OVERWRITTEN: fireUpdateEvent(result); break;
}
}
if (session.getTransactionManager().isActive()) {
session.getTransactionManager().commit();
}
return Response.ok(results).build();
} catch (ModelDuplicateException e) {
return ErrorResponse.exists(e.getLocalizedMessage());
} catch (ErrorResponseException error) {
if (session.getTransactionManager().isActive()) session.getTransactionManager().setRollbackOnly();
return error.getResponse();
} catch (Exception e) {
if (session.getTransactionManager().isActive()) session.getTransactionManager().setRollbackOnly();
return ErrorResponse.error(e.getMessage(), Response.Status.INTERNAL_SERVER_ERROR);
for (PartialImport partialImport : partialImports) {
partialImport.prepare(rep, realm, session);
}
}
private void fireCreatedEvent(PartialImportResult result) {
adminEvent.operation(OperationType.CREATE)
.resourcePath(result.getResourceType().getPath(), result.getId())
.representation(result.getRepresentation())
.success();
};
for (PartialImport partialImport : partialImports) {
partialImport.removeOverwrites(realm, session);
results.addAllResults(partialImport.doImport(rep, realm, session));
}
private void fireUpdateEvent(PartialImportResult result) {
adminEvent.operation(OperationType.UPDATE)
.resourcePath(result.getResourceType().getPath(), result.getId())
.representation(result.getRepresentation())
.success();
return results;
}
}

View file

@ -22,13 +22,15 @@ import org.keycloak.models.KeycloakSession;
import org.keycloak.models.KeycloakSessionFactory;
import org.keycloak.models.ModelException;
import org.keycloak.models.RealmModel;
import org.keycloak.partialimport.PartialImportManager;
import org.keycloak.provider.Provider;
import org.keycloak.provider.ProviderFactory;
import org.keycloak.storage.ImportRealmFromRepresentation;
import org.keycloak.storage.ImportRealmFromRepresentationEvent;
import org.keycloak.storage.PartialImportRealmFromRepresentationEvent;
import org.keycloak.storage.SetDefaultsForNewRealm;
/**
* Provider to listen for {@link org.keycloak.storage.ImportRealmFromRepresentation} events.
* Provider to listen for {@link ImportRealmFromRepresentationEvent} events.
* If that is no longer needed after further steps around the legacy storage migration, it can be removed.
*
* @author Alexander Schwartz
@ -47,10 +49,14 @@ public class RealmManagerProviderFactory implements ProviderFactory<RealmManager
@Override
public void postInit(KeycloakSessionFactory factory) {
factory.register(event -> {
if (event instanceof ImportRealmFromRepresentation) {
ImportRealmFromRepresentation importRealmFromRepresentation = (ImportRealmFromRepresentation) event;
RealmModel realmModel = new RealmManager(importRealmFromRepresentation.getSession()).importRealm(importRealmFromRepresentation.getRealmRepresentation());
importRealmFromRepresentation.setRealmModel(realmModel);
if (event instanceof ImportRealmFromRepresentationEvent) {
ImportRealmFromRepresentationEvent importRealmFromRepresentationEvent = (ImportRealmFromRepresentationEvent) event;
RealmModel realmModel = new RealmManager(importRealmFromRepresentationEvent.getSession()).importRealm(importRealmFromRepresentationEvent.getRealmRepresentation());
importRealmFromRepresentationEvent.setRealmModel(realmModel);
} else if (event instanceof PartialImportRealmFromRepresentationEvent) {
PartialImportRealmFromRepresentationEvent partialImportRealmFromRepresentationEvent = (PartialImportRealmFromRepresentationEvent) event;
PartialImportManager partialImportManager = new PartialImportManager(partialImportRealmFromRepresentationEvent.getRep(), partialImportRealmFromRepresentationEvent.getSession(), partialImportRealmFromRepresentationEvent.getRealm());
partialImportRealmFromRepresentationEvent.setPartialImportResults(partialImportManager.saveResources());
} else if (event instanceof SetDefaultsForNewRealm) {
SetDefaultsForNewRealm setDefaultsForNewRealm = (SetDefaultsForNewRealm) event;
new RealmManager(setDefaultsForNewRealm.getSession()).setDefaultsForNewRealm(setDefaultsForNewRealm.getRealmModel());

View file

@ -20,9 +20,10 @@ package org.keycloak.services.managers;
import org.keycloak.provider.Provider;
import org.keycloak.provider.ProviderFactory;
import org.keycloak.provider.Spi;
import org.keycloak.storage.ImportRealmFromRepresentationEvent;
/**
* Provider to listen for {@link org.keycloak.storage.ImportRealmFromRepresentation} events.
* Provider to listen for {@link ImportRealmFromRepresentationEvent} events.
* If that is no longer needed after further steps around the legacy storage migration, it can be removed.
*
* @author Alexander Schwartz

View file

@ -42,24 +42,52 @@ import java.util.function.Predicate;
public class AdminEventBuilder {
protected static final Logger logger = Logger.getLogger(AdminEventBuilder.class);
private final AdminAuth auth;
private final String ipAddress;
private final RealmModel realm;
private final AdminEvent adminEvent;
private final Map<String, EventListenerProvider> listeners;
private EventStoreProvider store;
private Map<String, EventListenerProvider> listeners;
private RealmModel realm;
private AdminEvent adminEvent;
public AdminEventBuilder(RealmModel realm, AdminAuth auth, KeycloakSession session, ClientConnection clientConnection) {
this(realm, auth, session, clientConnection.getRemoteAddr());
}
private AdminEventBuilder(RealmModel realm, AdminAuth auth, KeycloakSession session, String ipAddress) {
this.realm = realm;
adminEvent = new AdminEvent();
this.listeners = new HashMap<>();
updateStore(session);
addListeners(session);
this.auth = auth;
this.ipAddress = ipAddress;
realm(realm);
authRealm(auth.getRealm());
authClient(auth.getClient());
authUser(auth.getUser());
authIpAddress(clientConnection.getRemoteAddr());
authIpAddress(ipAddress);
}
/**
* Create a new instance of the {@link AdminEventBuilder} that is bound to a new session.
* Use this when starting, for example, a nested transaction.
* @param session new session where the {@link AdminEventBuilder} should be bound to.
* @return a new instance of {@link AdminEventBuilder}
*/
public AdminEventBuilder clone(KeycloakSession session) {
RealmModel newEventRealm = session.realms().getRealm(realm.getId());
RealmModel newAuthRealm = session.realms().getRealm(this.auth.getRealm().getId());
UserModel newAuthUser = session.users().getUserById(newAuthRealm, this.auth.getUser().getId());
ClientModel newAuthClient = session.clients().getClientById(newAuthRealm, this.auth.getClient().getId());
return new AdminEventBuilder(
newEventRealm,
new AdminAuth(newAuthRealm, this.auth.getToken(), newAuthUser, newAuthClient),
session,
ipAddress
);
}
public AdminEventBuilder realm(RealmModel realm) {

View file

@ -19,6 +19,7 @@ package org.keycloak.services.resources.admin;
import static org.keycloak.utils.LockObjectsForModification.lockUserSessionsForModification;
import static org.keycloak.util.JsonSerialization.readValue;
import java.io.InputStream;
import java.security.cert.X509Certificate;
import java.text.ParseException;
import java.text.SimpleDateFormat;
@ -87,7 +88,9 @@ import org.keycloak.models.utils.KeycloakModelUtils;
import org.keycloak.models.utils.ModelToRepresentation;
import org.keycloak.models.utils.RepresentationToModel;
import org.keycloak.models.utils.StripSecretsUtils;
import org.keycloak.partialimport.PartialImportManager;
import org.keycloak.partialimport.ErrorResponseException;
import org.keycloak.partialimport.PartialImportResult;
import org.keycloak.partialimport.PartialImportResults;
import org.keycloak.provider.InvalidationHandler;
import org.keycloak.representations.adapters.action.GlobalRequestResult;
import org.keycloak.representations.idm.AdminEventRepresentation;
@ -97,7 +100,6 @@ import org.keycloak.representations.idm.ComponentRepresentation;
import org.keycloak.representations.idm.EventRepresentation;
import org.keycloak.representations.idm.GroupRepresentation;
import org.keycloak.representations.idm.ManagementPermissionReference;
import org.keycloak.representations.idm.PartialImportRepresentation;
import org.keycloak.representations.idm.RealmEventsConfigRepresentation;
import org.keycloak.representations.idm.RealmRepresentation;
import org.keycloak.services.ErrorResponse;
@ -1005,17 +1007,54 @@ public class RealmAdminResource {
/**
* Partial import from a JSON file to an existing realm.
*
* @param rep
* @return
*/
@Path("partialImport")
@POST
@Consumes(MediaType.APPLICATION_JSON)
public Response partialImport(PartialImportRepresentation rep) {
public Response partialImport(InputStream requestBody) {
auth.realm().requireManageRealm();
try {
return Response.ok(
KeycloakModelUtils.runJobInTransactionWithResult(session.getKeycloakSessionFactory(), kcSession -> {
RealmModel realmClone = kcSession.realms().getRealm(realm.getId());
AdminEventBuilder adminEventClone = adminEvent.clone(kcSession);
// calling a static method to avoid using the wrong instances
return getPartialImportResults(requestBody, kcSession, realmClone, adminEventClone);
})
).build();
} catch (ModelDuplicateException e) {
return ErrorResponse.exists(e.getLocalizedMessage());
} catch (ErrorResponseException error) {
return error.getResponse();
} catch (Exception e) {
return ErrorResponse.error(e.getMessage(), Response.Status.INTERNAL_SERVER_ERROR);
}
}
PartialImportManager partialImport = new PartialImportManager(rep, session, realm, adminEvent);
return partialImport.saveResources();
private static PartialImportResults getPartialImportResults(InputStream requestBody, KeycloakSession kcSession, RealmModel kcRealm, AdminEventBuilder adminEventClone) {
ExportImportManager exportProvider = kcSession.getProvider(DatastoreProvider.class).getExportImportManager();
PartialImportResults results = exportProvider.partialImportRealm(kcRealm, requestBody);
for (PartialImportResult result : results.getResults()) {
switch (result.getAction()) {
case ADDED : fireCreatedEvent(result, adminEventClone); break;
case OVERWRITTEN: fireUpdateEvent(result, adminEventClone); break;
}
}
return results;
}
private static void fireCreatedEvent(PartialImportResult result, AdminEventBuilder adminEvent) {
adminEvent.operation(OperationType.CREATE)
.resourcePath(result.getResourceType().getPath(), result.getId())
.representation(result.getRepresentation())
.success();
};
private static void fireUpdateEvent(PartialImportResult result, AdminEventBuilder adminEvent) {
adminEvent.operation(OperationType.UPDATE)
.resourcePath(result.getResourceType().getPath(), result.getId())
.representation(result.getRepresentation())
.success();
}
/**