From f83b67cdf520608e329c1c9296765c7b33b13866 Mon Sep 17 00:00:00 2001 From: mposolda Date: Fri, 1 Apr 2016 15:33:39 +0200 Subject: [PATCH] KEYCLOAK-2413 Very slow export/import of realms with large users count --- .../en/en-US/modules/export-import.xml | 19 +++++++++---- .../exportimport/ExportImportConfig.java | 28 ++++--------------- .../exportimport/ExportImportManager.java | 4 +-- 3 files changed, 21 insertions(+), 30 deletions(-) diff --git a/docbook/auth-server-docs/reference/en/en-US/modules/export-import.xml b/docbook/auth-server-docs/reference/en/en-US/modules/export-import.xml index 3241d447e5..9a8a4bc1ff 100755 --- a/docbook/auth-server-docs/reference/en/en-US/modules/export-import.xml +++ b/docbook/auth-server-docs/reference/en/en-US/modules/export-import.xml @@ -39,12 +39,21 @@ - If you import to Directory, you can specify also the number of users to be stored in each JSON file. So if you have - very large amount of users in your database, you likely don't want to import them into single file as the file might be very big. - Processing of each file is done in separate transaction as exporting/importing all users at once could also lead to memory issues. + If you import to Directory, you can specify also the number of users to be stored in each JSON file. + + + If you have bigger amount of users in your database (500 or more), it's higly recommended to export into directory rather than to single file. + Exporting into single file may lead to the very big file. Also the directory provider is using separate transaction for each "page" (file with users), + which leads to much better performance. Default count of users per file (and transaction) is 50, which showed us best performance, but you have possibility to override (See below). + + + Exporting to single file is using one transaction per whole export and one per whole import, which leads to + bad performance with large amount of users - time increases exponentially with number of users. + + - To export into unencrypted directory you can use: + To export into the directory you can use: @@ -98,7 +107,7 @@ bin/standalone.sh -Dkeycloak.migration.action=import can be used to specify number of users per file (and also per DB transaction). - It's 5000 by default. It's used only if usersExportStrategy is DIFFERENT_FILES + It's 50 by default. It's used only if usersExportStrategy is DIFFERENT_FILES diff --git a/services/src/main/java/org/keycloak/exportimport/ExportImportConfig.java b/services/src/main/java/org/keycloak/exportimport/ExportImportConfig.java index 7de7aacf53..2555a19da7 100644 --- a/services/src/main/java/org/keycloak/exportimport/ExportImportConfig.java +++ b/services/src/main/java/org/keycloak/exportimport/ExportImportConfig.java @@ -28,26 +28,24 @@ public class ExportImportConfig { public static final String ACTION_IMPORT = "import"; public static final String PROVIDER = PREFIX + "provider"; - public static final String PROVIDER_DEFAULT = "zip"; + public static final String PROVIDER_DEFAULT = "dir"; // Name of the realm to export. If null, then full export will be triggered public static final String REALM_NAME = PREFIX + "realmName"; // used for "dir" provider public static final String DIR = PREFIX + "dir"; - // used for "zip" provider - public static final String ZIP_FILE = PREFIX + "zipFile"; - public static final String ZIP_PASSWORD = PREFIX + "zipPassword"; + // used for "singleFile" provider public static final String FILE = PREFIX + "file"; - // How to export users when realm export is requested for "dir" and "zip" provider + // How to export users when realm export is requested for "dir" provider public static final String USERS_EXPORT_STRATEGY = PREFIX + "usersExportStrategy"; public static final UsersExportStrategy DEFAULT_USERS_EXPORT_STRATEGY = UsersExportStrategy.DIFFERENT_FILES; - // Number of users per file used in "dir" and "zip" providers. Used if usersExportStrategy is DIFFERENT_FILES + // Number of users per file used in "dir" provider. Used if usersExportStrategy is DIFFERENT_FILES public static final String USERS_PER_FILE = PREFIX + "usersPerFile"; - public static final Integer DEFAULT_USERS_PER_FILE = 5000; + public static final Integer DEFAULT_USERS_PER_FILE = 50; // Strategy used during import data public static final String STRATEGY = PREFIX + "strategy"; @@ -89,22 +87,6 @@ public class ExportImportConfig { return System.setProperty(DIR, dir); } - public static String getZipFile() { - return System.getProperty(ZIP_FILE); - } - - public static void setZipFile(String exportImportZipFile) { - System.setProperty(ZIP_FILE, exportImportZipFile); - } - - public static String getZipPassword() { - return System.getProperty(ZIP_PASSWORD); - } - - public static void setZipPassword(String exportImportZipPassword) { - System.setProperty(ZIP_PASSWORD, exportImportZipPassword); - } - public static String getFile() { return System.getProperty(FILE); } diff --git a/services/src/main/java/org/keycloak/exportimport/ExportImportManager.java b/services/src/main/java/org/keycloak/exportimport/ExportImportManager.java index b06e8a20e2..f8fb7eac2f 100644 --- a/services/src/main/java/org/keycloak/exportimport/ExportImportManager.java +++ b/services/src/main/java/org/keycloak/exportimport/ExportImportManager.java @@ -49,12 +49,12 @@ public class ExportImportManager { if (ExportImportConfig.ACTION_EXPORT.equals(exportImportAction)) { exportProvider = session.getProvider(ExportProvider.class, providerId); if (exportProvider == null) { - throw new RuntimeException("Export provider not found"); + throw new RuntimeException("Export provider '" + providerId + "' not found"); } } else if (ExportImportConfig.ACTION_IMPORT.equals(exportImportAction)) { importProvider = session.getProvider(ImportProvider.class, providerId); if (importProvider == null) { - throw new RuntimeException("Import provider not found"); + throw new RuntimeException("Import provider '" + providerId + "' not found"); } } }