KEYCLOAK-2413 Very slow export/import of realms with large users count

docbook/auth-server-docs/reference/en/en-US/modules/export-import.xml

@@ -39,12 +39,21 @@
             </itemizedlist>
         </para>
         <para>
-            If you import to Directory, you can specify also the number of users to be stored in each JSON file. So if you have
+            If you import to Directory, you can specify also the number of users to be stored in each JSON file.
-            very large amount of users in your database, you likely don't want to import them into single file as the file might be very big.
-            Processing of each file is done in separate transaction as exporting/importing all users at once could also lead to memory issues.
         </para>
+        <warning>
+            <para>
+                If you have bigger amount of users in your database (500 or more), it's higly recommended to export into directory rather than to single file.
+                Exporting into single file may lead to the very big file. Also the directory provider is using separate transaction for each "page" (file with users),
+                which leads to much better performance. Default count of users per file (and transaction) is 50, which showed us best performance, but you have possibility to override (See below).
+            </para>
+            <para>
+                Exporting to single file is using one transaction per whole export and one per whole import, which leads to
+                bad performance with large amount of users - time increases exponentially with number of users.
+            </para>
+        </warning>
         <para>
-            To export into unencrypted directory you can use:
+            To export into the directory you can use:
             <programlisting><![CDATA[
 bin/standalone.sh -Dkeycloak.migration.action=export
 -Dkeycloak.migration.provider=dir -Dkeycloak.migration.dir=<DIR TO EXPORT TO>
@@ -98,7 +107,7 @@ bin/standalone.sh -Dkeycloak.migration.action=import
                     <listitem>
                         <para>
                             can be used to specify number of users per file (and also per DB transaction).
-                            It's 5000 by default. It's used only if usersExportStrategy is DIFFERENT_FILES
+                            It's 50 by default. It's used only if usersExportStrategy is DIFFERENT_FILES
                         </para>
                     </listitem>
                 </varlistentry>

services/src/main/java/org/keycloak/exportimport/ExportImportConfig.java

@@ -28,26 +28,24 @@ public class ExportImportConfig {
     public static final String ACTION_IMPORT = "import";
 
     public static final String PROVIDER = PREFIX + "provider";
-    public static final String PROVIDER_DEFAULT = "zip";
+    public static final String PROVIDER_DEFAULT = "dir";
 
     // Name of the realm to export. If null, then full export will be triggered
     public static final String REALM_NAME = PREFIX + "realmName";
 
     // used for "dir" provider
     public static final String DIR = PREFIX + "dir";
-    // used for "zip" provider
+
-    public static final String ZIP_FILE = PREFIX + "zipFile";
-    public static final String ZIP_PASSWORD = PREFIX + "zipPassword";
     // used for "singleFile" provider
     public static final String FILE = PREFIX + "file";
 
-    // How to export users when realm export is requested for "dir" and "zip" provider
+    // How to export users when realm export is requested for "dir" provider
     public static final String USERS_EXPORT_STRATEGY = PREFIX + "usersExportStrategy";
     public static final UsersExportStrategy DEFAULT_USERS_EXPORT_STRATEGY = UsersExportStrategy.DIFFERENT_FILES;
 
-    // Number of users per file used in "dir" and "zip" providers. Used if usersExportStrategy is DIFFERENT_FILES
+    // Number of users per file used in "dir" provider. Used if usersExportStrategy is DIFFERENT_FILES
     public static final String USERS_PER_FILE = PREFIX + "usersPerFile";
-    public static final Integer DEFAULT_USERS_PER_FILE = 5000;
+    public static final Integer DEFAULT_USERS_PER_FILE = 50;
 
     // Strategy used during import data
     public static final String STRATEGY = PREFIX + "strategy";
@@ -89,22 +87,6 @@ public class ExportImportConfig {
         return System.setProperty(DIR, dir);
     }
 
-    public static String getZipFile() {
-        return System.getProperty(ZIP_FILE);
-    }
-
-    public static void setZipFile(String exportImportZipFile) {
-        System.setProperty(ZIP_FILE, exportImportZipFile);
-    }
-
-    public static String getZipPassword() {
-        return System.getProperty(ZIP_PASSWORD);
-    }
-
-    public static void setZipPassword(String exportImportZipPassword) {
-        System.setProperty(ZIP_PASSWORD, exportImportZipPassword);
-    }
-
     public static String getFile() {
         return System.getProperty(FILE);
     }

services/src/main/java/org/keycloak/exportimport/ExportImportManager.java

@@ -49,12 +49,12 @@ public class ExportImportManager {
         if (ExportImportConfig.ACTION_EXPORT.equals(exportImportAction)) {
             exportProvider = session.getProvider(ExportProvider.class, providerId);
             if (exportProvider == null) {
-                throw new RuntimeException("Export provider not found");
+                throw new RuntimeException("Export provider '" + providerId + "' not found");
             }
         } else if (ExportImportConfig.ACTION_IMPORT.equals(exportImportAction)) {
             importProvider = session.getProvider(ImportProvider.class, providerId);
             if (importProvider == null) {
-                throw new RuntimeException("Import provider not found");
+                throw new RuntimeException("Import provider '" + providerId + "' not found");
             }
         }
     }