KEYCLOAK-17615 Move database initialization from KeycloakApplication to JpaConnectionProviderFactory

This commit is contained in:
vramik 2021-04-13 22:23:48 +02:00 committed by Hynek Mlnařík
parent 515bfb5064
commit 162043beec
16 changed files with 267 additions and 104 deletions

View file

@ -160,7 +160,7 @@ jobs:
run: |
declare -A PARAMS TESTGROUP
PARAMS["quarkus"]="-Pauth-server-quarkus"
PARAMS["undertow-map"]="-Pauth-server-undertow -Dkeycloak.client.provider=map -Dkeycloak.group.provider=map -Dkeycloak.role.provider=map -Dkeycloak.authSession.provider=map -Dkeycloak.userSession.provider=map -Dkeycloak.loginFailure.provider=map -Dkeycloak.user.provider=map -Dkeycloak.clientScope.provider=map -Dkeycloak.realm.provider=map -Dkeycloak.authorization.provider=map"
PARAMS["undertow-map"]="-Pauth-server-undertow -Dkeycloak.client.provider=map -Dkeycloak.group.provider=map -Dkeycloak.role.provider=map -Dkeycloak.authSession.provider=map -Dkeycloak.userSession.provider=map -Dkeycloak.loginFailure.provider=map -Dkeycloak.user.provider=map -Dkeycloak.clientScope.provider=map -Dkeycloak.realm.provider=map -Dkeycloak.authorization.provider=map -Dkeycloak.serverInfo.provider=map"
PARAMS["wildfly"]="-Pauth-server-wildfly"
TESTGROUP["group1"]="-Dtest=!**.crossdc.**,!**.cluster.**,%regex[org.keycloak.testsuite.(a[abc]|ad[a-l]|[^a-q]).*]" # Tests alphabetically before admin tests and those after "r"
TESTGROUP["group2"]="-Dtest=!**.crossdc.**,!**.cluster.**,%regex[org.keycloak.testsuite.(ad[^a-l]|a[^a-d]|b).*]" # Admin tests and those starting with "b"

View file

@ -25,6 +25,7 @@ import org.keycloak.ServerStartupError;
import org.keycloak.common.util.StringPropertyReplacer;
import org.keycloak.connections.jpa.updater.JpaUpdaterProvider;
import org.keycloak.connections.jpa.util.JpaUtils;
import org.keycloak.migration.MigrationModelManager;
import org.keycloak.models.KeycloakSession;
import org.keycloak.models.KeycloakSessionFactory;
import org.keycloak.models.KeycloakSessionTask;
@ -213,6 +214,18 @@ public class DefaultJpaConnectionProviderFactory implements JpaConnectionProvide
if (globalStatsInterval != -1) {
startGlobalStats(session, globalStatsInterval);
}
/*
* Migrate model is executed just in case following providers are "jpa".
* In Map Storage, there is an assumption that migrateModel is not needed.
*/
if ((Config.getProvider("realm") == null || "jpa".equals(Config.getProvider("realm"))) &&
(Config.getProvider("client") == null || "jpa".equals(Config.getProvider("client"))) &&
(Config.getProvider("clientScope") == null || "jpa".equals(Config.getProvider("clientScope")))) {
logger.debug("Calling migrateModel");
migrateModel(session);
}
} finally {
// Close after creating EntityManagerFactory to prevent in-mem databases from closing
if (connection != null) {
@ -402,4 +415,7 @@ public class DefaultJpaConnectionProviderFactory implements JpaConnectionProvide
}
}
private void migrateModel(KeycloakSession session) {
KeycloakModelUtils.runJobInTransaction(session.getKeycloakSessionFactory(), MigrationModelManager::migrate);
}
}

View file

@ -48,6 +48,7 @@ public class MigrationModelAdapter implements MigrationModel {
}
@Override
@Deprecated
public String getResourcesTag() {
return latest != null ? latest.getId() : null;
}

View file

@ -0,0 +1,87 @@
/*
* Copyright 2021 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.models.map.serverinfo;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import org.jboss.logging.Logger;
import org.keycloak.Config;
import org.keycloak.common.Version;
import org.keycloak.common.util.Base64Url;
import org.keycloak.common.util.RandomString;
import org.keycloak.migration.MigrationModel;
import org.keycloak.migration.ModelVersion;
import org.keycloak.models.KeycloakSession;
import org.keycloak.models.ServerInfoProvider;
import org.keycloak.models.ServerInfoProviderFactory;
import org.keycloak.models.map.common.AbstractMapProviderFactory;
public class MapServerInfoProviderFactory extends AbstractMapProviderFactory<ServerInfoProvider> implements ServerInfoProviderFactory {
private static final String RESOURCES_VERSION_SEED = "resourcesVersionSeed";
@Override
public void init(Config.Scope config) {
String seed = config.get(RESOURCES_VERSION_SEED);
if (seed == null) {
Logger.getLogger(ServerInfoProviderFactory.class).warnf("It is recommended to set '%s' property in the %s provider config of serverInfo SPI", RESOURCES_VERSION_SEED, PROVIDER_ID);
//generate random string for this installation
seed = RandomString.randomCode(10);
}
try {
Version.RESOURCES_VERSION = Base64Url.encode(MessageDigest.getInstance("MD5")
.digest((seed + new ModelVersion(Version.VERSION_KEYCLOAK).toString()).getBytes()))
.substring(0, 5);
} catch (NoSuchAlgorithmException e) {
throw new RuntimeException(e);
}
}
@Override
public ServerInfoProvider create(KeycloakSession session) {
return INSTANCE;
}
private static final ServerInfoProvider INSTANCE = new ServerInfoProvider() {
private final MigrationModel INSTANCE = new MigrationModel() {
@Override
public String getStoredVersion() {
return null;
}
@Override
public String getResourcesTag() {
throw new UnsupportedOperationException("Not supported.");
}
@Override
public void setStoredVersion(String version) {
throw new UnsupportedOperationException("Not supported.");
}
};
@Override
public MigrationModel getMigrationModel() {
return INSTANCE;
}
@Override
public void close() {
}
};
}

View file

@ -0,0 +1,18 @@
#
# Copyright 2021 Red Hat, Inc. and/or its affiliates
# and other contributors as indicated by the @author tags.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
org.keycloak.models.map.serverinfo.MapServerInfoProviderFactory

View file

@ -184,8 +184,18 @@ public final class QuarkusJpaConnectionProviderFactory implements JpaConnectionP
private void initSchemaOrExport(KeycloakSession session) {
ExportImportManager exportImportManager = new ExportImportManager(session);
logger.debug("Calling migrateModel");
migrateModel(session);
/*
* Migrate model is executed just in case following providers are "jpa".
* In Map Storage, there is an assumption that migrateModel is not needed.
*/
if ((Config.getProvider("realm") == null || "jpa".equals(Config.getProvider("realm"))) &&
(Config.getProvider("client") == null || "jpa".equals(Config.getProvider("client"))) &&
(Config.getProvider("clientScope") == null || "jpa".equals(Config.getProvider("clientScope")))) {
logger.debug("Calling migrateModel");
migrateModel(session);
}
DBLockManager dbLockManager = new DBLockManager(session);
dbLockManager.checkForcedUnlock();

View file

@ -24,6 +24,7 @@ package org.keycloak.migration;
*/
public interface MigrationModel {
String getStoredVersion();
@Deprecated
String getResourcesTag();
void setStoredVersion(String version);
}

View file

@ -22,7 +22,6 @@ import org.keycloak.Config;
import org.keycloak.common.util.Resteasy;
import org.keycloak.config.ConfigProviderFactory;
import org.keycloak.exportimport.ExportImportManager;
import org.keycloak.migration.MigrationModelManager;
import org.keycloak.models.KeycloakSession;
import org.keycloak.models.KeycloakSessionFactory;
import org.keycloak.models.KeycloakSessionTask;
@ -30,8 +29,6 @@ import org.keycloak.models.ModelDuplicateException;
import org.keycloak.models.RealmModel;
import org.keycloak.models.UserModel;
import org.keycloak.models.UserProvider;
import org.keycloak.models.dblock.DBLockManager;
import org.keycloak.models.dblock.DBLockProvider;
import org.keycloak.models.utils.KeycloakModelUtils;
import org.keycloak.models.utils.PostMigrationEvent;
import org.keycloak.models.utils.RepresentationToModel;
@ -86,8 +83,8 @@ public class KeycloakApplication extends Application {
protected final PlatformProvider platform = Platform.getPlatform();
protected Set<Object> singletons = new HashSet<Object>();
protected Set<Class<?>> classes = new HashSet<Class<?>>();
protected Set<Object> singletons = new HashSet<>();
protected Set<Class<?>> classes = new HashSet<>();
protected static KeycloakSessionFactory sessionFactory;
@ -124,28 +121,10 @@ public class KeycloakApplication extends Application {
protected void startup() {
this.sessionFactory = createSessionFactory();
ExportImportManager[] exportImportManager = new ExportImportManager[1];
ExportImportManager exportImportManager = bootstrap();
KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
@Override
public void run(KeycloakSession lockSession) {
DBLockManager dbLockManager = new DBLockManager(lockSession);
dbLockManager.checkForcedUnlock();
DBLockProvider dbLock = dbLockManager.getDBLock();
dbLock.waitForLock(DBLockProvider.Namespace.KEYCLOAK_BOOT);
try {
exportImportManager[0] = migrateAndBootstrap();
} finally {
dbLock.releaseLock();
}
}
});
if (exportImportManager[0].isRunExport()) {
exportImportManager[0].runExport();
if (exportImportManager.isRunExport()) {
exportImportManager.runExport();
}
KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
@ -168,78 +147,56 @@ public class KeycloakApplication extends Application {
sessionFactory.close();
}
// Migrate model, bootstrap master realm, import realms and create admin user. This is done with acquired dbLock
protected ExportImportManager migrateAndBootstrap() {
ExportImportManager exportImportManager;
logger.debug("Calling migrateModel");
migrateModel();
// Bootstrap master realm, import realms and create admin user.
protected ExportImportManager bootstrap() {
ExportImportManager[] exportImportManager = new ExportImportManager[1];
logger.debug("bootstrap");
KeycloakSession session = sessionFactory.create();
try {
session.getTransactionManager().begin();
JtaTransactionManagerLookup lookup = (JtaTransactionManagerLookup) sessionFactory.getProviderFactory(JtaTransactionManagerLookup.class);
if (lookup != null) {
if (lookup.getTransactionManager() != null) {
try {
Transaction transaction = lookup.getTransactionManager().getTransaction();
logger.debugv("bootstrap current transaction? {0}", transaction != null);
if (transaction != null) {
logger.debugv("bootstrap current transaction status? {0}", transaction.getStatus());
KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
@Override
public void run(KeycloakSession session) {
// TODO what is the purpose of following piece of code? Leaving it as is for now.
JtaTransactionManagerLookup lookup = (JtaTransactionManagerLookup) sessionFactory.getProviderFactory(JtaTransactionManagerLookup.class);
if (lookup != null) {
if (lookup.getTransactionManager() != null) {
try {
Transaction transaction = lookup.getTransactionManager().getTransaction();
logger.debugv("bootstrap current transaction? {0}", transaction != null);
if (transaction != null) {
logger.debugv("bootstrap current transaction status? {0}", transaction.getStatus());
}
} catch (SystemException e) {
throw new RuntimeException(e);
}
} catch (SystemException e) {
throw new RuntimeException(e);
}
}
// TODO up here ^^
session.clientPolicy().setupClientPoliciesOnKeycloakApp("/keycloak-default-client-profiles.json", "/keycloak-default-client-policies.json");
ApplianceBootstrap applianceBootstrap = new ApplianceBootstrap(session);
exportImportManager[0] = new ExportImportManager(session);
boolean createMasterRealm = applianceBootstrap.isNewInstall();
if (exportImportManager[0].isRunImport() && exportImportManager[0].isImportMasterIncluded()) {
createMasterRealm = false;
}
if (createMasterRealm) {
applianceBootstrap.createMasterRealm();
}
}
});
session.clientPolicy().setupClientPoliciesOnKeycloakApp("/keycloak-default-client-profiles.json", "/keycloak-default-client-policies.json");
ApplianceBootstrap applianceBootstrap = new ApplianceBootstrap(session);
exportImportManager = new ExportImportManager(session);
boolean createMasterRealm = applianceBootstrap.isNewInstall();
if (exportImportManager.isRunImport() && exportImportManager.isImportMasterIncluded()) {
createMasterRealm = false;
}
if (createMasterRealm) {
applianceBootstrap.createMasterRealm();
}
session.getTransactionManager().commit();
} catch (RuntimeException re) {
if (session.getTransactionManager().isActive()) {
session.getTransactionManager().rollback();
}
throw re;
} finally {
session.close();
}
if (exportImportManager.isRunImport()) {
exportImportManager.runImport();
if (exportImportManager[0].isRunImport()) {
exportImportManager[0].runImport();
} else {
importRealms();
}
importAddUser();
return exportImportManager;
}
protected void migrateModel() {
KeycloakSession session = sessionFactory.create();
try {
session.getTransactionManager().begin();
MigrationModelManager.migrate(session);
session.getTransactionManager().commit();
} catch (Exception e) {
session.getTransactionManager().rollback();
throw e;
} finally {
session.close();
}
return exportImportManager[0];
}
protected void loadConfig() {

View file

@ -66,7 +66,7 @@ public class ServerInfoTest extends AbstractKeycloakTest {
Map<String, ProviderRepresentation> jpaProviders = info.getProviders().get("connectionsJpa").getProviders();
ProviderRepresentation jpaProvider = jpaProviders.values().iterator().next();
log.infof("JPA Connections provider info: %s", jpaProvider.getOperationalInfo().toString());
log.infof("JPA Connections provider info: %s", jpaProvider.getOperationalInfo());
}
@Override

View file

@ -929,7 +929,7 @@ public abstract class AbstractMigrationTest extends AbstractKeycloakTest {
String response = SimpleHttp.doGet(url.toString(), client).asString();
Matcher m = Pattern.compile("resources/([^/]*)/welcome").matcher(response);
assertTrue(m.find());
assertTrue(m.group(1).matches("[\\da-z]{5}"));
assertTrue(m.group(1).matches("[a-zA-Z0-9_\\-.~]{5}"));
} catch (IOException e) {
fail(e.getMessage());
}

View file

@ -36,6 +36,13 @@
"event-queue": {}
},
"serverInfo": {
"provider": "${keycloak.serverInfo.provider:jpa}",
"map": {
"resourcesVersionSeed": "1JZ379bzyOCFA"
}
},
"realm": {
"provider": "${keycloak.realm.provider:jpa}"
},

View file

@ -71,6 +71,8 @@ import org.junit.rules.TestRule;
import org.junit.runner.Description;
import org.junit.runners.model.Statement;
import org.keycloak.timer.TimerSpi;
import org.keycloak.models.ServerInfoProviderFactory;
import org.keycloak.models.ServerInfoSpi;
/**
* Base of testcases that operate on session level. The tests derived from this class
@ -175,6 +177,7 @@ public abstract class KeycloakModelTest {
.add(GroupSpi.class)
.add(RealmSpi.class)
.add(RoleSpi.class)
.add(ServerInfoSpi.class)
.add(StoreFactorySpi.class)
.add(TimerSpi.class)
.add(UserLoginFailureSpi.class)
@ -185,6 +188,7 @@ public abstract class KeycloakModelTest {
private static final Set<Class<? extends ProviderFactory>> ALLOWED_FACTORIES = ImmutableSet.<Class<? extends ProviderFactory>>builder()
.add(DefaultAuthorizationProviderFactory.class)
.add(DefaultExecutorsProviderFactory.class)
.add(ServerInfoProviderFactory.class)
.build();
protected static final List<KeycloakModelParameters> MODEL_PARAMETERS;

View file

@ -1,33 +1,62 @@
/*
* Copyright 2021 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.testsuite.model;
import java.util.List;
import javax.persistence.EntityManager;
import org.jboss.logging.Logger;
import org.junit.Assert;
import org.junit.Test;
import org.keycloak.common.Version;
import org.keycloak.common.util.Time;
import org.keycloak.connections.jpa.JpaConnectionProvider;
import org.keycloak.migration.MigrationModel;
import org.keycloak.models.jpa.entities.MigrationModelEntity;
import org.keycloak.representations.idm.RealmRepresentation;
import org.keycloak.testsuite.AbstractKeycloakTest;
import org.keycloak.testsuite.arquillian.annotation.AuthServerContainerExclude;
import javax.persistence.EntityManager;
import java.util.List;
import org.jboss.logging.Logger;
import org.keycloak.models.ClientProvider;
import org.keycloak.models.ClientScopeProvider;
import org.keycloak.models.Constants;
import org.keycloak.models.KeycloakSession;
import org.keycloak.models.RealmModel;
import org.keycloak.models.RealmProvider;
import org.keycloak.models.ServerInfoProvider;
import org.keycloak.models.jpa.entities.MigrationModelEntity;
import static org.keycloak.testsuite.arquillian.annotation.AuthServerContainerExclude.AuthServer.REMOTE;
@RequireProvider(value=RealmProvider.class, only="jpa")
@RequireProvider(value=ClientProvider.class, only="jpa")
@RequireProvider(value=ClientScopeProvider.class, only="jpa")
public class MigrationModelTest extends KeycloakModelTest {
@AuthServerContainerExclude(REMOTE)
public class MigrationModelTest extends AbstractKeycloakTest {
private String realmId;
@Override
public void addTestRealms(List<RealmRepresentation> testRealms) {
public void createEnvironment(KeycloakSession s) {
RealmModel realm = s.realms().createRealm("realm");
realm.setDefaultRole(s.roles().addRealmRole(realm, Constants.DEFAULT_ROLES_ROLE_PREFIX + "-" + realm.getName()));
this.realmId = realm.getId();
}
@Override
public void cleanEnvironment(KeycloakSession s) {
s.realms().removeRealm(realmId);
}
@Test
public void test() {
testingClient.server().run(session -> {
inComittedTransaction(1, (session , i) -> {
String currentVersion = Version.VERSION_KEYCLOAK.replaceAll("^(\\d+(?:\\.\\d+){0,2}).*$", "$1");
JpaConnectionProvider p = session.getProvider(JpaConnectionProvider.class);
@ -64,7 +93,8 @@ public class MigrationModelTest extends AbstractKeycloakTest {
Assert.assertEquals(currentVersion, m.getStoredVersion());
em.remove(l.get(1));
return null;
});
}
}

View file

@ -27,6 +27,8 @@ import org.keycloak.connections.jpa.updater.liquibase.lock.LiquibaseDBLockProvid
import org.keycloak.events.jpa.JpaEventStoreProviderFactory;
import org.keycloak.models.jpa.session.JpaUserSessionPersisterProviderFactory;
import org.keycloak.models.session.UserSessionPersisterSpi;
import org.keycloak.migration.MigrationProviderFactory;
import org.keycloak.migration.MigrationSpi;
import org.keycloak.testsuite.model.KeycloakModelParameters;
import org.keycloak.models.dblock.DBLockSpi;
import org.keycloak.models.jpa.JpaClientProviderFactory;
@ -40,6 +42,8 @@ import org.keycloak.provider.Spi;
import org.keycloak.testsuite.model.Config;
import com.google.common.collect.ImmutableSet;
import java.util.Set;
import org.keycloak.protocol.LoginProtocolFactory;
import org.keycloak.protocol.LoginProtocolSpi;
/**
*
@ -55,6 +59,10 @@ public class Jpa extends KeycloakModelParameters {
.add(LiquibaseConnectionSpi.class)
.add(UserSessionPersisterSpi.class)
//required for migrateModel
.add(MigrationSpi.class)
.add(LoginProtocolSpi.class)
.build();
static final Set<Class<? extends ProviderFactory>> ALLOWED_FACTORIES = ImmutableSet.<Class<? extends ProviderFactory>>builder()
@ -72,6 +80,11 @@ public class Jpa extends KeycloakModelParameters {
.add(LiquibaseConnectionProviderFactory.class)
.add(LiquibaseDBLockProviderFactory.class)
.add(JpaUserSessionPersisterProviderFactory.class)
//required for migrateModel
.add(MigrationProviderFactory.class)
.add(LoginProtocolFactory.class)
.build();
public Jpa() {
@ -81,11 +94,17 @@ public class Jpa extends KeycloakModelParameters {
@Override
public void updateConfig(Config cf) {
updateConfigForJpa(cf);
}
public static void updateConfigForJpa(Config cf) {
cf.spi("client").defaultProvider("jpa")
.spi("clientScope").defaultProvider("jpa")
.spi("group").defaultProvider("jpa")
.spi("role").defaultProvider("jpa")
.spi("user").defaultProvider("jpa")
.spi("realm").defaultProvider("jpa")
.spi("serverInfo").defaultProvider("jpa")
;
}
}

View file

@ -31,6 +31,7 @@ import org.keycloak.storage.clientscope.ClientScopeStorageProviderFactory;
import org.keycloak.storage.clientscope.ClientScopeStorageProviderModel;
import org.keycloak.storage.clientscope.ClientScopeStorageProviderSpi;
import org.keycloak.testsuite.federation.HardcodedClientScopeStorageProviderFactory;
import org.keycloak.testsuite.model.Config;
/**
*
@ -70,4 +71,9 @@ public class JpaFederation extends KeycloakModelParameters {
return super.getParameters(clazz);
}
}
@Override
public void updateConfig(Config cf) {
Jpa.updateConfigForJpa(cf);
}
}

View file

@ -14,6 +14,13 @@
"provider": "${keycloak.eventsStore.provider:}"
},
"serverInfo": {
"provider": "${keycloak.serverInfo.provider:jpa}",
"map": {
"resourcesVersionSeed": "1JZ379bzyOCFA"
}
},
"realm": {
"provider": "${keycloak.realm.provider:jpa}"
},