Merge pull request #1443 from mposolda/master

JSON import issue from older versions. Fix import at startup if master realm not present

export-import/export-import-api/src/main/java/org/keycloak/exportimport/ImportProvider.java

@@ -13,4 +13,10 @@ public interface ImportProvider extends Provider {
     void importModel(KeycloakSessionFactory factory, Strategy strategy) throws IOException;
 
     void importRealm(KeycloakSessionFactory factory, String realmName, Strategy strategy) throws IOException;
+
+    /**
+     * @return true if master realm was previously exported and is available in the data to be imported
+     * @throws IOException
+     */
+    boolean isMasterRealmExported() throws IOException;
 }

export-import/export-import-api/src/main/java/org/keycloak/exportimport/util/ImportUtils.java

@@ -21,6 +21,7 @@ import org.keycloak.representations.idm.UserRepresentation;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import org.keycloak.exportimport.ExportImportConfig;
@@ -143,6 +144,15 @@ public class ImportUtils {
      * @throws IOException
      */
     public static void importFromStream(KeycloakSession session, ObjectMapper mapper, InputStream is, Strategy strategy) throws IOException {
+        Map<String, RealmRepresentation> realmReps = getRealmsFromStream(mapper, is);
+        for (RealmRepresentation realmRep : realmReps.values()) {
+            importRealm(session, realmRep, strategy);
+        }
+    }
+
+    public static Map<String, RealmRepresentation> getRealmsFromStream(ObjectMapper mapper, InputStream is) throws IOException {
+        Map<String, RealmRepresentation> result = new HashMap<String, RealmRepresentation>();
+
         JsonFactory factory = mapper.getJsonFactory();
         JsonParser parser = factory.createJsonParser(is);
         try {
@@ -166,18 +176,21 @@ public class ImportUtils {
                 }
 
                 for (RealmRepresentation realmRep : realmReps) {
-                    importRealm(session, realmRep, strategy);
+                    result.put(realmRep.getId(), realmRep);
                 }
             } else if (parser.getCurrentToken() == JsonToken.START_OBJECT) {
                 // Case with single realm in stream
                 RealmRepresentation realmRep = parser.readValueAs(RealmRepresentation.class);
-                importRealm(session, realmRep, strategy);
+                result.put(realmRep.getId(), realmRep);
             }
         } finally {
             parser.close();
         }
+
+        return result;
     }
 
+
     // Assuming that it's invoked inside transaction
     public static void importUsersFromStream(KeycloakSession session, String realmName, ObjectMapper mapper, InputStream is) throws IOException {
         RealmProvider model = session.realms();

export-import/export-import-dir/src/main/java/org/keycloak/exportimport/dir/DirImportProvider.java

@@ -49,6 +49,20 @@ public class DirImportProvider implements ImportProvider {
 
     @Override
     public void importModel(KeycloakSessionFactory factory, Strategy strategy) throws IOException {
+        List<String> realmNames = getRealmsToImport();
+
+        for (String realmName : realmNames) {
+            importRealm(factory, realmName, strategy);
+        }
+    }
+
+    @Override
+    public boolean isMasterRealmExported() throws IOException {
+        List<String> realmNames = getRealmsToImport();
+        return realmNames.contains(Config.getAdminRealm());
+    }
+
+    private List<String> getRealmsToImport() throws IOException {
         File[] realmFiles = this.rootDirectory.listFiles(new FilenameFilter() {
 
             @Override
@@ -70,10 +84,7 @@ public class DirImportProvider implements ImportProvider {
                 realmNames.add(realmName);
             }
         }
-
+        return realmNames;
-        for (String realmName : realmNames) {
-            importRealm(factory, realmName, strategy);
-        }
     }
 
     @Override

export-import/export-import-single-file/src/main/java/org/keycloak/exportimport/singlefile/SingleFileImportProvider.java

@@ -1,6 +1,7 @@
 package org.keycloak.exportimport.singlefile;
 
 import org.jboss.logging.Logger;
+import org.keycloak.Config;
 import org.keycloak.exportimport.ImportProvider;
 import org.keycloak.exportimport.Strategy;
 import org.keycloak.exportimport.util.ExportImportSessionTask;
@@ -8,11 +9,13 @@ import org.keycloak.exportimport.util.ImportUtils;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
 import org.keycloak.models.utils.KeycloakModelUtils;
+import org.keycloak.representations.idm.RealmRepresentation;
 import org.keycloak.util.JsonSerialization;
 
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.util.Map;
 
 /**
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
@@ -23,6 +26,9 @@ public class SingleFileImportProvider implements ImportProvider {
 
     private File file;
 
+    // Allows to cache representation per provider to avoid parsing them twice
+    protected Map<String, RealmRepresentation> realmReps;
+
     public SingleFileImportProvider(File file) {
         this.file = file;
     }
@@ -30,17 +36,33 @@ public class SingleFileImportProvider implements ImportProvider {
     @Override
     public void importModel(KeycloakSessionFactory factory, final Strategy strategy) throws IOException {
         logger.infof("Full importing from file %s", this.file.getAbsolutePath());
+        checkRealmReps();
+
         KeycloakModelUtils.runJobInTransaction(factory, new ExportImportSessionTask() {
 
             @Override
             protected void runExportImportTask(KeycloakSession session) throws IOException {
-                FileInputStream is = new FileInputStream(file);
+                for (RealmRepresentation realmRep : realmReps.values()) {
-                ImportUtils.importFromStream(session, JsonSerialization.mapper, is, strategy);
+                    ImportUtils.importRealm(session, realmRep, strategy);
+                }
             }
 
         });
     }
 
+    @Override
+    public boolean isMasterRealmExported() throws IOException {
+        checkRealmReps();
+        return (realmReps.containsKey(Config.getAdminRealm()));
+    }
+
+    protected void checkRealmReps() throws IOException {
+        if (realmReps == null) {
+            FileInputStream is = new FileInputStream(file);
+            realmReps = ImportUtils.getRealmsFromStream(JsonSerialization.mapper, is);
+        }
+    }
+
     @Override
     public void importRealm(KeycloakSessionFactory factory, String realmName, Strategy strategy) throws IOException {
         // TODO: import just that single realm in case that file contains many realms?

export-import/export-import-zip/src/main/java/org/keycloak/exportimport/zip/ZipImportProvider.java

@@ -51,6 +51,20 @@ public class ZipImportProvider implements ImportProvider {
 
     @Override
     public void importModel(KeycloakSessionFactory factory, Strategy strategy) throws IOException {
+        List<String> realmNames = getRealmsToImport();
+
+        for (String realmName : realmNames) {
+            importRealm(factory, realmName, strategy);
+        }
+    }
+
+    @Override
+    public boolean isMasterRealmExported() throws IOException {
+        List<String> realmNames = getRealmsToImport();
+        return realmNames.contains(Config.getAdminRealm());
+    }
+
+    private List<String> getRealmsToImport() throws IOException {
         List<String> realmNames = new ArrayList<String>();
         for (ExtZipEntry entry : this.decrypter.getEntryList()) {
             String entryName = entry.getName();
@@ -66,10 +80,7 @@ public class ZipImportProvider implements ImportProvider {
                 }
             }
         }
-
+        return realmNames;
-        for (String realmName : realmNames) {
-            importRealm(factory, realmName, strategy);
-        }
     }
 
     @Override

services/src/main/java/org/keycloak/exportimport/ExportImportManager.java

@@ -2,8 +2,10 @@ package org.keycloak.exportimport;
 
 
 import org.jboss.logging.Logger;
+import org.keycloak.Config;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
+import org.keycloak.services.managers.ApplianceBootstrap;
 
 /**
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
@@ -12,7 +14,7 @@ public class ExportImportManager {
 
     private static final Logger logger = Logger.getLogger(ExportImportManager.class);
 
-    public void checkExportImport(KeycloakSessionFactory sessionFactory) {
+    public void checkExportImport(KeycloakSessionFactory sessionFactory, String contextPath) {
         String exportImportAction = ExportImportConfig.getAction();
         String realmName = ExportImportConfig.getRealmName();
 
@@ -46,9 +48,21 @@ public class ExportImportManager {
                     Strategy strategy = ExportImportConfig.getStrategy();
                     if (realmName == null) {
                         logger.infof("Full model import requested. Strategy: %s", strategy.toString());
+
+                        // Check if master realm was exported. If it's not, then it needs to be created before other realms are imported
+                        if (!importProvider.isMasterRealmExported()) {
+                            new ApplianceBootstrap().bootstrap(sessionFactory, contextPath);
+                        }
+
                         importProvider.importModel(sessionFactory, strategy);
                     } else {
                         logger.infof("Import of realm '%s' requested. Strategy: %s", realmName, strategy.toString());
+
+                        if (!realmName.equals(Config.getAdminRealm())) {
+                            // Check if master realm exists. If it's not, then it needs to be created before other realm is imported
+                            new ApplianceBootstrap().bootstrap(sessionFactory, contextPath);
+                        }
+
                         importProvider.importRealm(sessionFactory, realmName, strategy);
                     }
                     logger.info("Import finished successfully");

services/src/main/java/org/keycloak/services/managers/RealmManager.java

@@ -21,7 +21,9 @@ import org.keycloak.models.utils.DefaultAuthenticationFlows;
 import org.keycloak.models.utils.DefaultRequiredActions;
 import org.keycloak.models.utils.KeycloakModelUtils;
 import org.keycloak.models.utils.RepresentationToModel;
+import org.keycloak.representations.idm.ApplicationRepresentation;
 import org.keycloak.representations.idm.ClientRepresentation;
+import org.keycloak.representations.idm.OAuthClientRepresentation;
 import org.keycloak.representations.idm.RealmEventsConfigRepresentation;
 import org.keycloak.representations.idm.RealmRepresentation;
 import org.keycloak.timer.TimerProvider;
@@ -267,13 +269,27 @@ public class RealmManager {
         setupMasterAdminManagement(realm);
         if (!hasRealmAdminManagementClient(rep)) setupRealmAdminManagement(realm);
         if (!hasAccountManagementClient(rep)) setupAccountManagement(realm);
-        if (!hasImpersonationServiceClient(rep)) setupImpersonationService(realm);
+
+        boolean postponeImpersonationSetup = false;
+        if (!hasImpersonationServiceClient(rep)) {
+            if (hasRealmAdminManagementClient(rep)) {
+                postponeImpersonationSetup = true;
+            } else {
+                setupImpersonationService(realm);
+            }
+        }
 
         if (!hasBrokerClient(rep)) setupBrokerService(realm);
         if (!hasAdminConsoleClient(rep)) setupAdminConsole(realm);
 
         RepresentationToModel.importRealm(session, rep, realm);
 
+        // Could happen when migrating from older version and I have exported JSON file, which contains "realm-management" client but not "impersonation" client
+        // I need to postpone impersonation because it needs "realm-management" client and it's roles set
+        if (postponeImpersonationSetup) {
+            setupImpersonationService(realm);
+        }
+
         setupAuthenticationFlows(realm);
         setupRequiredActions(realm);
 
@@ -287,50 +303,49 @@ public class RealmManager {
     }
 
     private boolean hasRealmAdminManagementClient(RealmRepresentation rep) {
-        if (rep.getClients() == null) return false;
+        String realmAdminClientId = getRealmAdminClientId(rep);
-        for (ClientRepresentation clientRep : rep.getClients()) {
+        return hasClient(rep, realmAdminClientId);
-            if (clientRep.getClientId().equals(getRealmAdminClientId(rep))) {
-                return true;
-            }
-        }
-        return false;
     }
 
     private boolean hasAccountManagementClient(RealmRepresentation rep) {
-        if (rep.getClients() == null) return false;
+        return hasClient(rep, Constants.ACCOUNT_MANAGEMENT_CLIENT_ID);
-        for (ClientRepresentation clientRep : rep.getClients()) {
-            if (clientRep.getClientId().equals(Constants.ACCOUNT_MANAGEMENT_CLIENT_ID)) {
-                return true;
-            }
-        }
-        return false;
     }
     private boolean hasImpersonationServiceClient(RealmRepresentation rep) {
-        if (rep.getClients() == null) return false;
+        return hasClient(rep, Constants.IMPERSONATION_SERVICE_CLIENT_ID);
-        for (ClientRepresentation clientRep : rep.getClients()) {
-            if (clientRep.getClientId().equals(Constants.IMPERSONATION_SERVICE_CLIENT_ID)) {
-                return true;
-            }
-        }
-        return false;
     }
     private boolean hasBrokerClient(RealmRepresentation rep) {
-        if (rep.getClients() == null) return false;
+        return hasClient(rep, Constants.BROKER_SERVICE_CLIENT_ID);
-        for (ClientRepresentation clientRep : rep.getClients()) {
-            if (clientRep.getClientId().equals(Constants.BROKER_SERVICE_CLIENT_ID)) {
-                return true;
-            }
-        }
-        return false;
     }
 
     private boolean hasAdminConsoleClient(RealmRepresentation rep) {
-        if (rep.getClients() == null) return false;
+        return hasClient(rep, Constants.ADMIN_CONSOLE_CLIENT_ID);
-        for (ClientRepresentation clientRep : rep.getClients()) {
+    }
-            if (clientRep.getClientId().equals(Constants.ADMIN_CONSOLE_CLIENT_ID)) {
+
-                return true;
+    private boolean hasClient(RealmRepresentation rep, String clientId) {
+        if (rep.getClients() != null) {
+            for (ClientRepresentation clientRep : rep.getClients()) {
+                if (clientRep.getClientId().equals(clientId)) {
+                    return true;
+                }
             }
         }
+
+        // TODO: Just for compatibility with old versions. Should be removed later...
+        if (rep.getApplications() != null) {
+            for (ApplicationRepresentation clientRep : rep.getApplications()) {
+                if (clientRep.getName().equals(clientId)) {
+                    return true;
+                }
+            }
+        }
+        if (rep.getOauthClients() != null) {
+            for (OAuthClientRepresentation clientRep : rep.getOauthClients()) {
+                if (clientRep.getName().equals(clientId)) {
+                    return true;
+                }
+            }
+        }
+
         return false;
     }
 

services/src/main/java/org/keycloak/services/resources/KeycloakApplication.java

@@ -83,7 +83,7 @@ public class KeycloakApplication extends Application {
         classes.add(JsResource.class);
         classes.add(WelcomeResource.class);
 
-        new ExportImportManager().checkExportImport(this.sessionFactory);
+        new ExportImportManager().checkExportImport(this.sessionFactory, context.getContextPath());
 
         setupDefaultRealm(context.getContextPath());
 

services/src/main/java/org/keycloak/services/resources/admin/RealmsAdminResource.java

@@ -168,12 +168,8 @@ public class RealmsAdminResource {
         for (InputPart inputPart : inputParts) {
             // inputPart.getBody doesn't work as content-type is wrong, and inputPart.setMediaType is not supported on AS7 (RestEasy 2.3.2.Final)
             rep = JsonSerialization.readValue(inputPart.getBodyAsString(), RealmRepresentation.class);
-            RealmModel realm;
+
-            try {
+            RealmModel realm = realmManager.importRealm(rep);
-                realm = realmManager.importRealm(rep);
-            } catch (ModelDuplicateException e) {
-                return ErrorResponse.exists("Realm " + rep.getRealm() + " already exists");
-            }
 
             grantPermissionsToRealmCreator(realm);