libre.sh/keycloak-scim
3
0
Fork 0
Code Issues 15 Pull requests Releases Packages Activity Actions

KEYCLOAK-2413 Very slow export/import of realms with large users count

Browse source
This commit is contained in:
mposolda 2016-04-01 15:33:39 +02:00
parent 46f6b97c26
commit f83b67cdf5
3 changed files with 21 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
docbook/auth-server-docs/reference/en/en-US/modules/export-import.xml
View file

@ -39,12 +39,21 @@
</itemizedlist>
</para>
<para>
If you import to Directory, you can specify also the number of users to be stored in each JSON file. So if you have
very large amount of users in your database, you likely don't want to import them into single file as the file might be very big.
Processing of each file is done in separate transaction as exporting/importing all users at once could also lead to memory issues.
If you import to Directory, you can specify also the number of users to be stored in each JSON file.
</para>
<warning>
<para>
If you have bigger amount of users in your database (500 or more), it's higly recommended to export into directory rather than to single file.
Exporting into single file may lead to the very big file. Also the directory provider is using separate transaction for each "page" (file with users),
which leads to much better performance. Default count of users per file (and transaction) is 50, which showed us best performance, but you have possibility to override (See below).
</para>
<para>
Exporting to single file is using one transaction per whole export and one per whole import, which leads to
bad performance with large amount of users - time increases exponentially with number of users.
</para>
</warning>
<para>
To export into unencrypted directory you can use:
To export into the directory you can use:
<programlisting><![CDATA[
bin/standalone.sh -Dkeycloak.migration.action=export
-Dkeycloak.migration.provider=dir -Dkeycloak.migration.dir=<DIR TO EXPORT TO>
@ -98,7 +107,7 @@ bin/standalone.sh -Dkeycloak.migration.action=import
<listitem>
<para>
can be used to specify number of users per file (and also per DB transaction).
It's 5000 by default. It's used only if usersExportStrategy is DIFFERENT_FILES
It's 50 by default. It's used only if usersExportStrategy is DIFFERENT_FILES
</para>
</listitem>
</varlistentry>

28
services/src/main/java/org/keycloak/exportimport/ExportImportConfig.java
View file

@ -28,26 +28,24 @@ public class ExportImportConfig {
public static final String ACTION_IMPORT = "import";
public static final String PROVIDER = PREFIX + "provider";
public static final String PROVIDER_DEFAULT = "zip";
public static final String PROVIDER_DEFAULT = "dir";
// Name of the realm to export. If null, then full export will be triggered
public static final String REALM_NAME = PREFIX + "realmName";
// used for "dir" provider
public static final String DIR = PREFIX + "dir";
// used for "zip" provider
public static final String ZIP_FILE = PREFIX + "zipFile";
public static final String ZIP_PASSWORD = PREFIX + "zipPassword";
// used for "singleFile" provider
public static final String FILE = PREFIX + "file";
// How to export users when realm export is requested for "dir" and "zip" provider
// How to export users when realm export is requested for "dir" provider
public static final String USERS_EXPORT_STRATEGY = PREFIX + "usersExportStrategy";
public static final UsersExportStrategy DEFAULT_USERS_EXPORT_STRATEGY = UsersExportStrategy.DIFFERENT_FILES;
// Number of users per file used in "dir" and "zip" providers. Used if usersExportStrategy is DIFFERENT_FILES
// Number of users per file used in "dir" provider. Used if usersExportStrategy is DIFFERENT_FILES
public static final String USERS_PER_FILE = PREFIX + "usersPerFile";
public static final Integer DEFAULT_USERS_PER_FILE = 5000;
public static final Integer DEFAULT_USERS_PER_FILE = 50;
// Strategy used during import data
public static final String STRATEGY = PREFIX + "strategy";
@ -89,22 +87,6 @@ public class ExportImportConfig {
return System.setProperty(DIR, dir);
}
public static String getZipFile() {
return System.getProperty(ZIP_FILE);
}
public static void setZipFile(String exportImportZipFile) {
System.setProperty(ZIP_FILE, exportImportZipFile);
}
public static String getZipPassword() {
return System.getProperty(ZIP_PASSWORD);
}
public static void setZipPassword(String exportImportZipPassword) {
System.setProperty(ZIP_PASSWORD, exportImportZipPassword);
}
public static String getFile() {
return System.getProperty(FILE);
}

4
services/src/main/java/org/keycloak/exportimport/ExportImportManager.java
View file

@ -49,12 +49,12 @@ public class ExportImportManager {
if (ExportImportConfig.ACTION_EXPORT.equals(exportImportAction)) {
exportProvider = session.getProvider(ExportProvider.class, providerId);
if (exportProvider == null) {
throw new RuntimeException("Export provider not found");
throw new RuntimeException("Export provider '" + providerId + "' not found");
}
} else if (ExportImportConfig.ACTION_IMPORT.equals(exportImportAction)) {
importProvider = session.getProvider(ImportProvider.class, providerId);
if (importProvider == null) {
throw new RuntimeException("Import provider not found");
throw new RuntimeException("Import provider '" + providerId + "' not found");
}
}
}