libre.sh/keycloak-scim
3
0
Fork 0
Code Issues 15 Pull requests Releases Packages Activity Actions

Merge pull request #2468 from mposolda/master

Browse source 
KEYCLOAK-2613 KEYCLOAK-2413 Export/import fixes and improvements
This commit is contained in:
Marek Posolda 2016-04-01 16:46:17 +02:00
commit e02d0ed629
8 changed files with 39 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
core/src/main/java/org/keycloak/representations/idm/RealmRepresentation.java
View file

@ -121,6 +121,8 @@ public class RealmRepresentation {
protected String resetCredentialsFlow;
protected String clientAuthenticationFlow;
protected String keycloakVersion;
@Deprecated
protected Boolean social;
@Deprecated
@ -814,6 +816,14 @@ public class RealmRepresentation {
this.clientAuthenticationFlow = clientAuthenticationFlow;
}
public String getKeycloakVersion() {
return keycloakVersion;
}
public void setKeycloakVersion(String keycloakVersion) {
this.keycloakVersion = keycloakVersion;
}
public List<GroupRepresentation> getGroups() {
return groups;
}

19
docbook/auth-server-docs/reference/en/en-US/modules/export-import.xml
View file

@ -39,12 +39,21 @@
</itemizedlist>
</para>
<para>
If you import to Directory, you can specify also the number of users to be stored in each JSON file. So if you have
very large amount of users in your database, you likely don't want to import them into single file as the file might be very big.
Processing of each file is done in separate transaction as exporting/importing all users at once could also lead to memory issues.
If you import to Directory, you can specify also the number of users to be stored in each JSON file.
</para>
<warning>
<para>
If you have bigger amount of users in your database (500 or more), it's higly recommended to export into directory rather than to single file.
Exporting into single file may lead to the very big file. Also the directory provider is using separate transaction for each "page" (file with users),
which leads to much better performance. Default count of users per file (and transaction) is 50, which showed us best performance, but you have possibility to override (See below).
</para>
<para>
Exporting to single file is using one transaction per whole export and one per whole import, which leads to
bad performance with large amount of users - time increases exponentially with number of users.
</para>
</warning>
<para>
To export into unencrypted directory you can use:
To export into the directory you can use:
<programlisting><![CDATA[
bin/standalone.sh -Dkeycloak.migration.action=export
-Dkeycloak.migration.provider=dir -Dkeycloak.migration.dir=<DIR TO EXPORT TO>
@ -98,7 +107,7 @@ bin/standalone.sh -Dkeycloak.migration.action=import
<listitem>
<para>
can be used to specify number of users per file (and also per DB transaction).
It's 5000 by default. It's used only if usersExportStrategy is DIFFERENT_FILES
It's 50 by default. It's used only if usersExportStrategy is DIFFERENT_FILES
</para>
</listitem>
</varlistentry>

28
services/src/main/java/org/keycloak/exportimport/ExportImportConfig.java
View file

@ -28,26 +28,24 @@ public class ExportImportConfig {
public static final String ACTION_IMPORT = "import";
public static final String PROVIDER = PREFIX + "provider";
public static final String PROVIDER_DEFAULT = "zip";
public static final String PROVIDER_DEFAULT = "dir";
// Name of the realm to export. If null, then full export will be triggered
public static final String REALM_NAME = PREFIX + "realmName";
// used for "dir" provider
public static final String DIR = PREFIX + "dir";
// used for "zip" provider
public static final String ZIP_FILE = PREFIX + "zipFile";
public static final String ZIP_PASSWORD = PREFIX + "zipPassword";
// used for "singleFile" provider
public static final String FILE = PREFIX + "file";
// How to export users when realm export is requested for "dir" and "zip" provider
// How to export users when realm export is requested for "dir" provider
public static final String USERS_EXPORT_STRATEGY = PREFIX + "usersExportStrategy";
public static final UsersExportStrategy DEFAULT_USERS_EXPORT_STRATEGY = UsersExportStrategy.DIFFERENT_FILES;
// Number of users per file used in "dir" and "zip" providers. Used if usersExportStrategy is DIFFERENT_FILES
// Number of users per file used in "dir" provider. Used if usersExportStrategy is DIFFERENT_FILES
public static final String USERS_PER_FILE = PREFIX + "usersPerFile";
public static final Integer DEFAULT_USERS_PER_FILE = 5000;
public static final Integer DEFAULT_USERS_PER_FILE = 50;
// Strategy used during import data
public static final String STRATEGY = PREFIX + "strategy";
@ -89,22 +87,6 @@ public class ExportImportConfig {
return System.setProperty(DIR, dir);
}
public static String getZipFile() {
return System.getProperty(ZIP_FILE);
}
public static void setZipFile(String exportImportZipFile) {
System.setProperty(ZIP_FILE, exportImportZipFile);
}
public static String getZipPassword() {
return System.getProperty(ZIP_PASSWORD);
}
public static void setZipPassword(String exportImportZipPassword) {
System.setProperty(ZIP_PASSWORD, exportImportZipPassword);
}
public static String getFile() {
return System.getProperty(FILE);
}

4
services/src/main/java/org/keycloak/exportimport/ExportImportManager.java
View file

@ -49,12 +49,12 @@ public class ExportImportManager {
if (ExportImportConfig.ACTION_EXPORT.equals(exportImportAction)) {
exportProvider = session.getProvider(ExportProvider.class, providerId);
if (exportProvider == null) {
throw new RuntimeException("Export provider not found");
throw new RuntimeException("Export provider '" + providerId + "' not found");
}
} else if (ExportImportConfig.ACTION_IMPORT.equals(exportImportAction)) {
importProvider = session.getProvider(ImportProvider.class, providerId);
if (importProvider == null) {
throw new RuntimeException("Import provider not found");
throw new RuntimeException("Import provider '" + providerId + "' not found");
}
}
}

8
services/src/main/java/org/keycloak/exportimport/dir/DirExportProvider.java
View file

@ -17,7 +17,6 @@
package org.keycloak.exportimport.dir;
import org.keycloak.representations.VersionRepresentation;
import org.keycloak.exportimport.util.ExportUtils;
import org.keycloak.exportimport.util.MultipleStepsExportProvider;
import org.keycloak.models.KeycloakSession;
@ -86,13 +85,6 @@ public class DirExportProvider extends MultipleStepsExportProvider {
ExportUtils.exportUsersToStream(session, realm, users, JsonSerialization.prettyMapper, os);
}
@Override
protected void writeVersion(String fileName, VersionRepresentation version) throws IOException {
File file = new File(this.rootDirectory, fileName);
FileOutputStream stream = new FileOutputStream(file);
JsonSerialization.prettyMapper.writeValue(stream, version);
}
@Override
public void close() {
}

4
services/src/main/java/org/keycloak/exportimport/util/ExportUtils.java
View file

@ -22,6 +22,7 @@ import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import org.keycloak.common.Version;
import org.keycloak.common.util.Base64;
import org.keycloak.models.*;
import org.keycloak.models.utils.ModelToRepresentation;
@ -39,6 +40,9 @@ public class ExportUtils {
public static RealmRepresentation exportRealm(KeycloakSession session, RealmModel realm, boolean includeUsers) {
RealmRepresentation rep = ModelToRepresentation.toRepresentation(realm, true);
// Project/product version
rep.setKeycloakVersion(Version.VERSION);
// Client Templates
List<ClientTemplateModel> templates = realm.getClientTemplates();
List<ClientTemplateRepresentation> templateReps = new ArrayList<>();

6
services/src/main/java/org/keycloak/exportimport/util/MultipleStepsExportProvider.java
View file

@ -18,7 +18,6 @@
package org.keycloak.exportimport.util;
import org.jboss.logging.Logger;
import org.keycloak.representations.VersionRepresentation;
import org.keycloak.exportimport.ExportImportConfig;
import org.keycloak.exportimport.ExportProvider;
import org.keycloak.exportimport.UsersExportStrategy;
@ -57,14 +56,11 @@ public abstract class MultipleStepsExportProvider implements ExportProvider {
for (RealmModel realm : holder.realms) {
exportRealmImpl(factory, realm.getName());
}
writeVersion("version.json", VersionRepresentation.SINGLETON);
}
@Override
public void exportRealm(KeycloakSessionFactory factory, String realmName) throws IOException {
exportRealmImpl(factory, realmName);
writeVersion("version.json", VersionRepresentation.SINGLETON);
}
protected void exportRealmImpl(KeycloakSessionFactory factory, final String realmName) throws IOException {
@ -127,8 +123,6 @@ public abstract class MultipleStepsExportProvider implements ExportProvider {
protected abstract void writeUsers(String fileName, KeycloakSession session, RealmModel realm, List<UserModel> users) throws IOException;
protected abstract void writeVersion(String fileName, VersionRepresentation version) throws IOException;
public static class RealmsHolder {
List<RealmModel> realms;

8
testsuite/integration/src/test/java/org/keycloak/testsuite/exportimport/ExportImportTest.java
View file

@ -166,8 +166,8 @@ public class ExportImportTest {
testFullExportImport();
// There should be 6 files in target directory (3 realm, 3 user, 1 version)
Assert.assertEquals(7, new File(targetDirPath).listFiles().length);
// There should be 6 files in target directory (3 realm, 3 user)
Assert.assertEquals(6, new File(targetDirPath).listFiles().length);
}
@Test
@ -180,9 +180,9 @@ public class ExportImportTest {
testRealmExportImport();
// There should be 3 files in target directory (1 realm, 3 user, 1 version)
// There should be 3 files in target directory (1 realm, 3 user)
File[] files = new File(targetDirPath).listFiles();
Assert.assertEquals(5, files.length);
Assert.assertEquals(4, files.length);
}
@Test