libre.sh/keycloak-scim
2
0
Fork 0
Code Issues 14 Pull requests Releases Packages Activity Actions 6

Merge pull request #535 from mposolda/master

Browse source 
Export/import improvements and fixes
This commit is contained in:
Bill Burke 2014-07-17 16:34:39 -04:00
commit 573f7fc1fb
10 changed files with 101 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
export-import/export-import-api/src/main/java/org/keycloak/exportimport/ExportImportConfig.java
View file

@ -24,7 +24,11 @@ public class ExportImportConfig {
// used for "singleFile" provider
public static final String FILE = PREFIX + "file";
// Number of users per file used in "dir" and "zip" providers. -1 means adding users to same file with realm. 0 means adding to separate file with unlimited page number
// How to export users when realm export is requested for "dir" and "zip" provider
public static final String USERS_EXPORT_STRATEGY = PREFIX + "usersExportStrategy";
public static final UsersExportStrategy DEFAULT_USERS_EXPORT_STRATEGY = UsersExportStrategy.DIFFERENT_FILES;
// Number of users per file used in "dir" and "zip" providers. Used if usersExportStrategy is DIFFERENT_FILES
public static final String USERS_PER_FILE = PREFIX + "usersPerFile";
public static final Integer DEFAULT_USERS_PER_FILE = 5000;
@ -92,6 +96,15 @@ public class ExportImportConfig {
System.setProperty(FILE, file);
}
public static UsersExportStrategy getUsersExportStrategy() {
String usersExportStrategy = System.getProperty(USERS_EXPORT_STRATEGY, DEFAULT_USERS_EXPORT_STRATEGY.toString());
return Enum.valueOf(UsersExportStrategy.class, usersExportStrategy);
}
public static void setUsersExportStrategy(UsersExportStrategy usersExportStrategy) {
System.setProperty(USERS_EXPORT_STRATEGY, usersExportStrategy.toString());
}
public static Integer getUsersPerFile() {
String usersPerFile = System.getProperty(USERS_PER_FILE, String.valueOf(DEFAULT_USERS_PER_FILE));
return Integer.parseInt(usersPerFile.trim());

11
export-import/export-import-api/src/main/java/org/keycloak/exportimport/UsersExportStrategy.java Normal file
View file

@ -0,0 +1,11 @@
package org.keycloak.exportimport;
/**
* @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
*/
public enum UsersExportStrategy {
SKIP, // Exporting of users will be skipped completely
REALM_FILE, // All users will be exported to same file with realm (So file like "foo-realm.json" with both realm data and users)
SAME_FILE, // All users will be exported to same file but different than realm (So file like "foo-realm.json" with realm data and "foo-users.json" with users)
DIFFERENT_FILES // Users will be exported into more different files according to maximum number of users per file
}

18
export-import/export-import-api/src/main/java/org/keycloak/exportimport/util/ImportUtils.java
View file

@ -51,8 +51,12 @@ public class ImportUtils {
} else {
logger.infof("Realm '%s' already exists. Removing it before import", realmName);
if (Config.getAdminRealm().equals(realm.getId())) {
realm.setMasterAdminApp(null);
// Delete all masterAdmin apps due to foreign key constraints
for (RealmModel currRealm : model.getRealms()) {
currRealm.setMasterAdminApp(null);
}
}
// TODO: For migration between versions, it should be possible to delete just realm but keep it's users
model.removeRealm(realm.getId());
}
}
@ -138,9 +142,21 @@ public class ImportUtils {
if (parser.getCurrentToken() == JsonToken.START_ARRAY) {
// Case with more realms in stream
parser.nextToken();
List<RealmRepresentation> realmReps = new ArrayList<RealmRepresentation>();
while (parser.getCurrentToken() == JsonToken.START_OBJECT) {
RealmRepresentation realmRep = parser.readValueAs(RealmRepresentation.class);
parser.nextToken();
// Ensure that master realm is imported first
if (Config.getAdminRealm().equals(realmRep.getRealm())) {
realmReps.add(0, realmRep);
} else {
realmReps.add(realmRep);
}
}
for (RealmRepresentation realmRep : realmReps) {
importRealm(session, realmRep, strategy);
}
} else if (parser.getCurrentToken() == JsonToken.START_OBJECT) {

17
export-import/export-import-api/src/main/java/org/keycloak/exportimport/util/MultipleStepsExportProvider.java
View file

@ -6,6 +6,7 @@ import java.util.List;
import org.jboss.logging.Logger;
import org.keycloak.exportimport.ExportImportConfig;
import org.keycloak.exportimport.ExportProvider;
import org.keycloak.exportimport.UsersExportStrategy;
import org.keycloak.models.KeycloakSession;
import org.keycloak.models.KeycloakSessionFactory;
import org.keycloak.models.RealmModel;
@ -23,7 +24,6 @@ public abstract class MultipleStepsExportProvider implements ExportProvider {
public void exportModel(KeycloakSessionFactory factory) throws IOException {
final RealmsHolder holder = new RealmsHolder();
// Import users into same file with realm
ExportImportUtils.runJobInTransaction(factory, new ExportImportJob() {
@Override
@ -41,33 +41,34 @@ public abstract class MultipleStepsExportProvider implements ExportProvider {
@Override
public void exportRealm(KeycloakSessionFactory factory, final String realmName) throws IOException {
final UsersExportStrategy usersExportStrategy = ExportImportConfig.getUsersExportStrategy();
final int usersPerFile = ExportImportConfig.getUsersPerFile();
final UsersHolder usersHolder = new UsersHolder();
final boolean exportUsersIntoSameFile = usersPerFile < 0;
final boolean exportUsersIntoRealmFile = usersExportStrategy == UsersExportStrategy.REALM_FILE;
ExportImportUtils.runJobInTransaction(factory, new ExportImportJob() {
@Override
public void run(KeycloakSession session) throws IOException {
RealmModel realm = session.realms().getRealmByName(realmName);
RealmRepresentation rep = ExportUtils.exportRealm(session, realm, exportUsersIntoSameFile);
RealmRepresentation rep = ExportUtils.exportRealm(session, realm, exportUsersIntoRealmFile);
writeRealm(realmName + "-realm.json", rep);
logger.info("Realm '" + realmName + "' - data exported");
// Count total number of users
if (!exportUsersIntoSameFile) {
if (!exportUsersIntoRealmFile) {
usersHolder.totalCount = session.users().getUsersCount(realm);
}
}
});
if (!exportUsersIntoSameFile) {
if (usersExportStrategy != UsersExportStrategy.SKIP && !exportUsersIntoRealmFile) {
// We need to export users now
usersHolder.currentPageStart = 0;
// usersPerFile==0 means exporting all users into single file (but separate to realm)
final int countPerPage = usersPerFile == 0 ? usersHolder.totalCount : usersPerFile;
// usersExportStrategy==SAME_FILE means exporting all users into single file (but separate to realm)
final int countPerPage = (usersExportStrategy == UsersExportStrategy.SAME_FILE) ? usersHolder.totalCount : usersPerFile;
while (usersHolder.currentPageStart < usersHolder.totalCount) {
if (usersHolder.currentPageStart + countPerPage < usersHolder.totalCount) {

18
export-import/export-import-dir/src/main/java/org/keycloak/exportimport/dir/DirImportProvider.java
View file

@ -4,8 +4,14 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import org.jboss.logging.Logger;
import org.keycloak.Config;
import org.keycloak.exportimport.ImportProvider;
import org.keycloak.exportimport.Strategy;
import org.keycloak.exportimport.util.ExportImportJob;
@ -54,11 +60,21 @@ public class DirImportProvider implements ImportProvider {
}
});
List<String> realmNames = new ArrayList<String>();
for (File file : realmFiles) {
String fileName = file.getName();
// Parse "foo" from "foo-realm.json"
String realmName = fileName.substring(0, fileName.length() - 11);
// Ensure that master realm is imported first
if (Config.getAdminRealm().equals(realmName)) {
realmNames.add(0, realmName);
} else {
realmNames.add(realmName);
}
}
for (String realmName : realmNames) {
importRealm(factory, realmName, strategy);
}
}

16
export-import/export-import-zip/src/main/java/org/keycloak/exportimport/zip/ZipImportProvider.java
View file

@ -4,6 +4,8 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.zip.DataFormatException;
import de.idyl.winzipaes.AesZipFileDecrypter;
@ -11,6 +13,7 @@ import de.idyl.winzipaes.impl.AESDecrypter;
import de.idyl.winzipaes.impl.AESDecrypterBC;
import de.idyl.winzipaes.impl.ExtZipEntry;
import org.jboss.logging.Logger;
import org.keycloak.Config;
import org.keycloak.exportimport.ImportProvider;
import org.keycloak.exportimport.Strategy;
import org.keycloak.exportimport.util.ExportImportJob;
@ -48,16 +51,27 @@ public class ZipImportProvider implements ImportProvider {
@Override
public void importModel(KeycloakSessionFactory factory, Strategy strategy) throws IOException {
List<String> realmNames = new ArrayList<String>();
for (ExtZipEntry entry : this.decrypter.getEntryList()) {
String entryName = entry.getName();
if (entryName.endsWith("-realm.json")) {
// Parse "foo" from "foo-realm.json"
String realmName = entryName.substring(0, entryName.length() - 11);
importRealm(factory, realmName, strategy);
// Ensure that master realm is imported first
if (Config.getAdminRealm().equals(realmName)) {
realmNames.add(0, realmName);
} else {
realmNames.add(realmName);
}
}
}
for (String realmName : realmNames) {
importRealm(factory, realmName, strategy);
}
}
@Override
public void importRealm(KeycloakSessionFactory factory, final String realmName, final Strategy strategy) throws IOException {
try {

22
model/api/src/main/java/org/keycloak/models/utils/RepresentationToModel.java
View file

@ -118,10 +118,7 @@ public class RepresentationToModel {
}
for (RoleRepresentation roleRep : entry.getValue()) {
// Application role may already exists (for example if it is defaultRole)
RoleModel role = app.getRole(roleRep.getName());
if (role == null) {
role = app.addRole(roleRep.getName());
}
RoleModel role = roleRep.getId()!=null ? app.addRole(roleRep.getId(), roleRep.getName()) : app.addRole(roleRep.getName());
role.setDescription(roleRep.getDescription());
}
}
@ -147,12 +144,21 @@ public class RepresentationToModel {
}
}
// Setup realm default roles
if (rep.getDefaultRoles() != null) {
for (String roleString : rep.getDefaultRoles()) {
newRealm.addDefaultRole(roleString.trim());
}
}
// Setup application default roles
if (rep.getApplications() != null) {
for (ApplicationRepresentation resourceRep : rep.getApplications()) {
if (resourceRep.getDefaultRoles() != null) {
ApplicationModel appModel = newRealm.getApplicationByName(resourceRep.getName());
appModel.updateDefaultRoles(resourceRep.getDefaultRoles());
}
}
}
if (rep.getOauthClients() != null) {
createOAuthClients(rep, newRealm);
@ -336,7 +342,7 @@ public class RepresentationToModel {
private static Map<String, ApplicationModel> createApplications(RealmRepresentation rep, RealmModel realm) {
Map<String, ApplicationModel> appMap = new HashMap<String, ApplicationModel>();
for (ApplicationRepresentation resourceRep : rep.getApplications()) {
ApplicationModel app = createApplication(realm, resourceRep);
ApplicationModel app = createApplication(realm, resourceRep, false);
appMap.put(app.getName(), app);
}
return appMap;
@ -349,7 +355,7 @@ public class RepresentationToModel {
* @param resourceRep
* @return
*/
public static ApplicationModel createApplication(RealmModel realm, ApplicationRepresentation resourceRep) {
public static ApplicationModel createApplication(RealmModel realm, ApplicationRepresentation resourceRep, boolean addDefaultRoles) {
logger.debug("************ CREATE APPLICATION: {0}" + resourceRep.getName());
ApplicationModel applicationModel = resourceRep.getId()!=null ? realm.addApplication(resourceRep.getId(), resourceRep.getName()) : realm.addApplication(resourceRep.getName());
if (resourceRep.isEnabled() != null) applicationModel.setEnabled(resourceRep.isEnabled());
@ -403,7 +409,7 @@ public class RepresentationToModel {
}
}
if (resourceRep.getDefaultRoles() != null) {
if (addDefaultRoles && resourceRep.getDefaultRoles() != null) {
applicationModel.updateDefaultRoles(resourceRep.getDefaultRoles());
}

2
model/tests/src/test/java/org/keycloak/model/test/ApplicationModelTest.java
View file

@ -61,7 +61,7 @@ public class ApplicationModelTest extends AbstractModelTest {
ApplicationRepresentation representation = ModelToRepresentation.toRepresentation(application);
RealmModel realm = realmManager.createRealm("copy");
ApplicationModel copy = RepresentationToModel.createApplication(realm, representation);
ApplicationModel copy = RepresentationToModel.createApplication(realm, representation, true);
assertEquals(application, copy);
}

2
services/src/main/java/org/keycloak/services/resources/admin/ApplicationsResource.java
View file

@ -87,7 +87,7 @@ public class ApplicationsResource {
auth.requireManage();
try {
ApplicationModel applicationModel = RepresentationToModel.createApplication(realm, rep);
ApplicationModel applicationModel = RepresentationToModel.createApplication(realm, rep, true);
return Response.created(uriInfo.getAbsolutePathBuilder().path(applicationModel.getName()).build()).build();
} catch (ModelDuplicateException e) {
return Flows.errors().exists("Application " + rep.getName() + " already exists");

4
testsuite/integration/src/test/java/org/keycloak/testsuite/exportimport/ExportImportTest.java
View file

@ -81,7 +81,7 @@ public class ExportImportTest {
@Override
protected void after() {
if (previousMongoClearOnStartup != null) {
System.setProperty(MONGO_CLEAR_ON_STARTUP_PROP_NAME, "false");
System.setProperty(MONGO_CLEAR_ON_STARTUP_PROP_NAME, previousMongoClearOnStartup);
} else {
System.getProperties().remove(MONGO_CLEAR_ON_STARTUP_PROP_NAME);
}
@ -128,7 +128,7 @@ public class ExportImportTest {
.around(mongoRule)
.around(keycloakRule);
//@Test
@Test
public void testDirFullExportImport() throws Throwable {
ExportImportConfig.setProvider(DirExportProviderFactory.PROVIDER_ID);
String targetDirPath = getExportImportTestDirectory() + File.separator + "dirExport";