libre.sh/keycloak-scim
3
0
Fork 0
Code Issues 15 Pull requests Releases 1 Packages Activity Actions

KEYCLOAK-17615 Move database initialization from KeycloakApplication to JpaConnectionProviderFactory

Browse source
This commit is contained in:
vramik 2021-04-13 22:23:48 +02:00 committed by Hynek Mlnařík
parent 515bfb5064
commit 162043beec
16 changed files with 267 additions and 104 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/ci.yml vendored
View file

@ -160,7 +160,7 @@ jobs:
run: |
declare -A PARAMS TESTGROUP
PARAMS["quarkus"]="-Pauth-server-quarkus"
PARAMS["undertow-map"]="-Pauth-server-undertow -Dkeycloak.client.provider=map -Dkeycloak.group.provider=map -Dkeycloak.role.provider=map -Dkeycloak.authSession.provider=map -Dkeycloak.userSession.provider=map -Dkeycloak.loginFailure.provider=map -Dkeycloak.user.provider=map -Dkeycloak.clientScope.provider=map -Dkeycloak.realm.provider=map -Dkeycloak.authorization.provider=map"
PARAMS["undertow-map"]="-Pauth-server-undertow -Dkeycloak.client.provider=map -Dkeycloak.group.provider=map -Dkeycloak.role.provider=map -Dkeycloak.authSession.provider=map -Dkeycloak.userSession.provider=map -Dkeycloak.loginFailure.provider=map -Dkeycloak.user.provider=map -Dkeycloak.clientScope.provider=map -Dkeycloak.realm.provider=map -Dkeycloak.authorization.provider=map -Dkeycloak.serverInfo.provider=map"
PARAMS["wildfly"]="-Pauth-server-wildfly"
TESTGROUP["group1"]="-Dtest=!**.crossdc.**,!**.cluster.**,%regex[org.keycloak.testsuite.(a[abc]|ad[a-l]|[^a-q]).*]" # Tests alphabetically before admin tests and those after "r"
TESTGROUP["group2"]="-Dtest=!**.crossdc.**,!**.cluster.**,%regex[org.keycloak.testsuite.(ad[^a-l]|a[^a-d]|b).*]" # Admin tests and those starting with "b"

16
model/jpa/src/main/java/org/keycloak/connections/jpa/DefaultJpaConnectionProviderFactory.java
View file

@ -25,6 +25,7 @@ import org.keycloak.ServerStartupError;
import org.keycloak.common.util.StringPropertyReplacer;
import org.keycloak.connections.jpa.updater.JpaUpdaterProvider;
import org.keycloak.connections.jpa.util.JpaUtils;
import org.keycloak.migration.MigrationModelManager;
import org.keycloak.models.KeycloakSession;
import org.keycloak.models.KeycloakSessionFactory;
import org.keycloak.models.KeycloakSessionTask;
@ -213,6 +214,18 @@ public class DefaultJpaConnectionProviderFactory implements JpaConnectionProvide
if (globalStatsInterval != -1) {
startGlobalStats(session, globalStatsInterval);
}
/*
* Migrate model is executed just in case following providers are "jpa".
* In Map Storage, there is an assumption that migrateModel is not needed.
*/
if ((Config.getProvider("realm") == null || "jpa".equals(Config.getProvider("realm"))) &&
(Config.getProvider("client") == null || "jpa".equals(Config.getProvider("client"))) &&
(Config.getProvider("clientScope") == null || "jpa".equals(Config.getProvider("clientScope")))) {
logger.debug("Calling migrateModel");
migrateModel(session);
}
} finally {
// Close after creating EntityManagerFactory to prevent in-mem databases from closing
if (connection != null) {
@ -402,4 +415,7 @@ public class DefaultJpaConnectionProviderFactory implements JpaConnectionProvide
}
}
private void migrateModel(KeycloakSession session) {
KeycloakModelUtils.runJobInTransaction(session.getKeycloakSessionFactory(), MigrationModelManager::migrate);
}
}

1
model/jpa/src/main/java/org/keycloak/models/jpa/MigrationModelAdapter.java
View file

@ -48,6 +48,7 @@ public class MigrationModelAdapter implements MigrationModel {
}
@Override
@Deprecated
public String getResourcesTag() {
return latest != null ? latest.getId() : null;
}

87
model/map/src/main/java/org/keycloak/models/map/serverinfo/MapServerInfoProviderFactory.java Normal file
View file

@ -0,0 +1,87 @@
/*
* Copyright 2021 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.models.map.serverinfo;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import org.jboss.logging.Logger;
import org.keycloak.Config;
import org.keycloak.common.Version;
import org.keycloak.common.util.Base64Url;
import org.keycloak.common.util.RandomString;
import org.keycloak.migration.MigrationModel;
import org.keycloak.migration.ModelVersion;
import org.keycloak.models.KeycloakSession;
import org.keycloak.models.ServerInfoProvider;
import org.keycloak.models.ServerInfoProviderFactory;
import org.keycloak.models.map.common.AbstractMapProviderFactory;
public class MapServerInfoProviderFactory extends AbstractMapProviderFactory<ServerInfoProvider> implements ServerInfoProviderFactory {
private static final String RESOURCES_VERSION_SEED = "resourcesVersionSeed";
@Override
public void init(Config.Scope config) {
String seed = config.get(RESOURCES_VERSION_SEED);
if (seed == null) {
Logger.getLogger(ServerInfoProviderFactory.class).warnf("It is recommended to set '%s' property in the %s provider config of serverInfo SPI", RESOURCES_VERSION_SEED, PROVIDER_ID);
//generate random string for this installation
seed = RandomString.randomCode(10);
}
try {
Version.RESOURCES_VERSION = Base64Url.encode(MessageDigest.getInstance("MD5")
.digest((seed + new ModelVersion(Version.VERSION_KEYCLOAK).toString()).getBytes()))
.substring(0, 5);
} catch (NoSuchAlgorithmException e) {
throw new RuntimeException(e);
}
}
@Override
public ServerInfoProvider create(KeycloakSession session) {
return INSTANCE;
}
private static final ServerInfoProvider INSTANCE = new ServerInfoProvider() {
private final MigrationModel INSTANCE = new MigrationModel() {
@Override
public String getStoredVersion() {
return null;
}
@Override
public String getResourcesTag() {
throw new UnsupportedOperationException("Not supported.");
}
@Override
public void setStoredVersion(String version) {
throw new UnsupportedOperationException("Not supported.");
}
};
@Override
public MigrationModel getMigrationModel() {
return INSTANCE;
}
@Override
public void close() {
}
};
}

18
model/map/src/main/resources/META-INF/services/org.keycloak.models.ServerInfoProviderFactory Normal file
View file

@ -0,0 +1,18 @@
#
# Copyright 2021 Red Hat, Inc. and/or its affiliates
# and other contributors as indicated by the @author tags.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
org.keycloak.models.map.serverinfo.MapServerInfoProviderFactory

10
quarkus/runtime/src/main/java/org/keycloak/connections/jpa/QuarkusJpaConnectionProviderFactory.java
View file

@ -184,8 +184,18 @@ public final class QuarkusJpaConnectionProviderFactory implements JpaConnectionP
private void initSchemaOrExport(KeycloakSession session) {
ExportImportManager exportImportManager = new ExportImportManager(session);
/*
* Migrate model is executed just in case following providers are "jpa".
* In Map Storage, there is an assumption that migrateModel is not needed.
*/
if ((Config.getProvider("realm") == null || "jpa".equals(Config.getProvider("realm"))) &&
(Config.getProvider("client") == null || "jpa".equals(Config.getProvider("client"))) &&
(Config.getProvider("clientScope") == null || "jpa".equals(Config.getProvider("clientScope")))) {
logger.debug("Calling migrateModel");
migrateModel(session);
}
DBLockManager dbLockManager = new DBLockManager(session);
dbLockManager.checkForcedUnlock();

1
server-spi/src/main/java/org/keycloak/migration/MigrationModel.java
View file

@ -24,6 +24,7 @@ package org.keycloak.migration;
*/
public interface MigrationModel {
String getStoredVersion();
@Deprecated
String getResourcesTag();
void setStoredVersion(String version);
}

81
services/src/main/java/org/keycloak/services/resources/KeycloakApplication.java
View file

@ -22,7 +22,6 @@ import org.keycloak.Config;
import org.keycloak.common.util.Resteasy;
import org.keycloak.config.ConfigProviderFactory;
import org.keycloak.exportimport.ExportImportManager;
import org.keycloak.migration.MigrationModelManager;
import org.keycloak.models.KeycloakSession;
import org.keycloak.models.KeycloakSessionFactory;
import org.keycloak.models.KeycloakSessionTask;
@ -30,8 +29,6 @@ import org.keycloak.models.ModelDuplicateException;
import org.keycloak.models.RealmModel;
import org.keycloak.models.UserModel;
import org.keycloak.models.UserProvider;
import org.keycloak.models.dblock.DBLockManager;
import org.keycloak.models.dblock.DBLockProvider;
import org.keycloak.models.utils.KeycloakModelUtils;
import org.keycloak.models.utils.PostMigrationEvent;
import org.keycloak.models.utils.RepresentationToModel;
@ -86,8 +83,8 @@ public class KeycloakApplication extends Application {
protected final PlatformProvider platform = Platform.getPlatform();
protected Set<Object> singletons = new HashSet<Object>();
protected Set<Class<?>> classes = new HashSet<Class<?>>();
protected Set<Object> singletons = new HashSet<>();
protected Set<Class<?>> classes = new HashSet<>();
protected static KeycloakSessionFactory sessionFactory;
@ -124,28 +121,10 @@ public class KeycloakApplication extends Application {
protected void startup() {
this.sessionFactory = createSessionFactory();
ExportImportManager[] exportImportManager = new ExportImportManager[1];
ExportImportManager exportImportManager = bootstrap();
KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
@Override
public void run(KeycloakSession lockSession) {
DBLockManager dbLockManager = new DBLockManager(lockSession);
dbLockManager.checkForcedUnlock();
DBLockProvider dbLock = dbLockManager.getDBLock();
dbLock.waitForLock(DBLockProvider.Namespace.KEYCLOAK_BOOT);
try {
exportImportManager[0] = migrateAndBootstrap();
} finally {
dbLock.releaseLock();
}
}
});
if (exportImportManager[0].isRunExport()) {
exportImportManager[0].runExport();
if (exportImportManager.isRunExport()) {
exportImportManager.runExport();
}
KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
@ -168,16 +147,15 @@ public class KeycloakApplication extends Application {
sessionFactory.close();
}
// Migrate model, bootstrap master realm, import realms and create admin user. This is done with acquired dbLock
protected ExportImportManager migrateAndBootstrap() {
ExportImportManager exportImportManager;
logger.debug("Calling migrateModel");
migrateModel();
// Bootstrap master realm, import realms and create admin user.
protected ExportImportManager bootstrap() {
ExportImportManager[] exportImportManager = new ExportImportManager[1];
logger.debug("bootstrap");
KeycloakSession session = sessionFactory.create();
try {
session.getTransactionManager().begin();
KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
@Override
public void run(KeycloakSession session) {
// TODO what is the purpose of following piece of code? Leaving it as is for now.
JtaTransactionManagerLookup lookup = (JtaTransactionManagerLookup) sessionFactory.getProviderFactory(JtaTransactionManagerLookup.class);
if (lookup != null) {
if (lookup.getTransactionManager() != null) {
@ -192,54 +170,33 @@ public class KeycloakApplication extends Application {
}
}
}
// TODO up here ^^
session.clientPolicy().setupClientPoliciesOnKeycloakApp("/keycloak-default-client-profiles.json", "/keycloak-default-client-policies.json");
ApplianceBootstrap applianceBootstrap = new ApplianceBootstrap(session);
exportImportManager = new ExportImportManager(session);
exportImportManager[0] = new ExportImportManager(session);
boolean createMasterRealm = applianceBootstrap.isNewInstall();
if (exportImportManager.isRunImport() && exportImportManager.isImportMasterIncluded()) {
if (exportImportManager[0].isRunImport() && exportImportManager[0].isImportMasterIncluded()) {
createMasterRealm = false;
}
if (createMasterRealm) {
applianceBootstrap.createMasterRealm();
}
session.getTransactionManager().commit();
} catch (RuntimeException re) {
if (session.getTransactionManager().isActive()) {
session.getTransactionManager().rollback();
}
throw re;
} finally {
session.close();
}
});
if (exportImportManager.isRunImport()) {
exportImportManager.runImport();
if (exportImportManager[0].isRunImport()) {
exportImportManager[0].runImport();
} else {
importRealms();
}
importAddUser();
return exportImportManager;
}
protected void migrateModel() {
KeycloakSession session = sessionFactory.create();
try {
session.getTransactionManager().begin();
MigrationModelManager.migrate(session);
session.getTransactionManager().commit();
} catch (Exception e) {
session.getTransactionManager().rollback();
throw e;
} finally {
session.close();
}
return exportImportManager[0];
}
protected void loadConfig() {

2
testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/admin/ServerInfoTest.java
View file

@ -66,7 +66,7 @@ public class ServerInfoTest extends AbstractKeycloakTest {
Map<String, ProviderRepresentation> jpaProviders = info.getProviders().get("connectionsJpa").getProviders();
ProviderRepresentation jpaProvider = jpaProviders.values().iterator().next();
log.infof("JPA Connections provider info: %s", jpaProvider.getOperationalInfo().toString());
log.infof("JPA Connections provider info: %s", jpaProvider.getOperationalInfo());
}
@Override

2
testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/migration/AbstractMigrationTest.java
View file

@ -929,7 +929,7 @@ public abstract class AbstractMigrationTest extends AbstractKeycloakTest {
String response = SimpleHttp.doGet(url.toString(), client).asString();
Matcher m = Pattern.compile("resources/([^/]*)/welcome").matcher(response);
assertTrue(m.find());
assertTrue(m.group(1).matches("[\\da-z]{5}"));
assertTrue(m.group(1).matches("[a-zA-Z0-9_\\-.~]{5}"));
} catch (IOException e) {
fail(e.getMessage());
}

7
testsuite/integration-arquillian/tests/base/src/test/resources/META-INF/keycloak-server.json
View file

@ -36,6 +36,13 @@
"event-queue": {}
},
"serverInfo": {
"provider": "${keycloak.serverInfo.provider:jpa}",
"map": {
"resourcesVersionSeed": "1JZ379bzyOCFA"
}
},
"realm": {
"provider": "${keycloak.realm.provider:jpa}"
},

4
testsuite/model/src/test/java/org/keycloak/testsuite/model/KeycloakModelTest.java
View file

@ -71,6 +71,8 @@ import org.junit.rules.TestRule;
import org.junit.runner.Description;
import org.junit.runners.model.Statement;
import org.keycloak.timer.TimerSpi;
import org.keycloak.models.ServerInfoProviderFactory;
import org.keycloak.models.ServerInfoSpi;
/**
* Base of testcases that operate on session level. The tests derived from this class
@ -175,6 +177,7 @@ public abstract class KeycloakModelTest {
.add(GroupSpi.class)
.add(RealmSpi.class)
.add(RoleSpi.class)
.add(ServerInfoSpi.class)
.add(StoreFactorySpi.class)
.add(TimerSpi.class)
.add(UserLoginFailureSpi.class)
@ -185,6 +188,7 @@ public abstract class KeycloakModelTest {
private static final Set<Class<? extends ProviderFactory>> ALLOWED_FACTORIES = ImmutableSet.<Class<? extends ProviderFactory>>builder()
.add(DefaultAuthorizationProviderFactory.class)
.add(DefaultExecutorsProviderFactory.class)
.add(ServerInfoProviderFactory.class)
.build();
protected static final List<KeycloakModelParameters> MODEL_PARAMETERS;

58
testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/model/MigrationModelTest.java → testsuite/model/src/test/java/org/keycloak/testsuite/model/MigrationModelTest.java
View file

@ -1,33 +1,62 @@
/*
* Copyright 2021 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.testsuite.model;
import java.util.List;
import javax.persistence.EntityManager;
import org.jboss.logging.Logger;
import org.junit.Assert;
import org.junit.Test;
import org.keycloak.common.Version;
import org.keycloak.common.util.Time;
import org.keycloak.connections.jpa.JpaConnectionProvider;
import org.keycloak.migration.MigrationModel;
import org.keycloak.models.jpa.entities.MigrationModelEntity;
import org.keycloak.representations.idm.RealmRepresentation;
import org.keycloak.testsuite.AbstractKeycloakTest;
import org.keycloak.testsuite.arquillian.annotation.AuthServerContainerExclude;
import javax.persistence.EntityManager;
import java.util.List;
import org.jboss.logging.Logger;
import org.keycloak.models.ClientProvider;
import org.keycloak.models.ClientScopeProvider;
import org.keycloak.models.Constants;
import org.keycloak.models.KeycloakSession;
import org.keycloak.models.RealmModel;
import org.keycloak.models.RealmProvider;
import org.keycloak.models.ServerInfoProvider;
import org.keycloak.models.jpa.entities.MigrationModelEntity;
import static org.keycloak.testsuite.arquillian.annotation.AuthServerContainerExclude.AuthServer.REMOTE;
@RequireProvider(value=RealmProvider.class, only="jpa")
@RequireProvider(value=ClientProvider.class, only="jpa")
@RequireProvider(value=ClientScopeProvider.class, only="jpa")
public class MigrationModelTest extends KeycloakModelTest {
@AuthServerContainerExclude(REMOTE)
public class MigrationModelTest extends AbstractKeycloakTest {
private String realmId;
@Override
public void addTestRealms(List<RealmRepresentation> testRealms) {
public void createEnvironment(KeycloakSession s) {
RealmModel realm = s.realms().createRealm("realm");
realm.setDefaultRole(s.roles().addRealmRole(realm, Constants.DEFAULT_ROLES_ROLE_PREFIX + "-" + realm.getName()));
this.realmId = realm.getId();
}
@Override
public void cleanEnvironment(KeycloakSession s) {
s.realms().removeRealm(realmId);
}
@Test
public void test() {
testingClient.server().run(session -> {
inComittedTransaction(1, (session , i) -> {
String currentVersion = Version.VERSION_KEYCLOAK.replaceAll("^(\\d+(?:\\.\\d+){0,2}).*$", "$1");
JpaConnectionProvider p = session.getProvider(JpaConnectionProvider.class);
@ -64,7 +93,8 @@ public class MigrationModelTest extends AbstractKeycloakTest {
Assert.assertEquals(currentVersion, m.getStoredVersion());
em.remove(l.get(1));
return null;
});
}
}

19
testsuite/model/src/test/java/org/keycloak/testsuite/model/parameters/Jpa.java
View file

@ -27,6 +27,8 @@ import org.keycloak.connections.jpa.updater.liquibase.lock.LiquibaseDBLockProvid
import org.keycloak.events.jpa.JpaEventStoreProviderFactory;
import org.keycloak.models.jpa.session.JpaUserSessionPersisterProviderFactory;
import org.keycloak.models.session.UserSessionPersisterSpi;
import org.keycloak.migration.MigrationProviderFactory;
import org.keycloak.migration.MigrationSpi;
import org.keycloak.testsuite.model.KeycloakModelParameters;
import org.keycloak.models.dblock.DBLockSpi;
import org.keycloak.models.jpa.JpaClientProviderFactory;
@ -40,6 +42,8 @@ import org.keycloak.provider.Spi;
import org.keycloak.testsuite.model.Config;
import com.google.common.collect.ImmutableSet;
import java.util.Set;
import org.keycloak.protocol.LoginProtocolFactory;
import org.keycloak.protocol.LoginProtocolSpi;
/**
*
@ -55,6 +59,10 @@ public class Jpa extends KeycloakModelParameters {
.add(LiquibaseConnectionSpi.class)
.add(UserSessionPersisterSpi.class)
//required for migrateModel
.add(MigrationSpi.class)
.add(LoginProtocolSpi.class)
.build();
static final Set<Class<? extends ProviderFactory>> ALLOWED_FACTORIES = ImmutableSet.<Class<? extends ProviderFactory>>builder()
@ -72,6 +80,11 @@ public class Jpa extends KeycloakModelParameters {
.add(LiquibaseConnectionProviderFactory.class)
.add(LiquibaseDBLockProviderFactory.class)
.add(JpaUserSessionPersisterProviderFactory.class)
//required for migrateModel
.add(MigrationProviderFactory.class)
.add(LoginProtocolFactory.class)
.build();
public Jpa() {
@ -81,11 +94,17 @@ public class Jpa extends KeycloakModelParameters {
@Override
public void updateConfig(Config cf) {
updateConfigForJpa(cf);
}
public static void updateConfigForJpa(Config cf) {
cf.spi("client").defaultProvider("jpa")
.spi("clientScope").defaultProvider("jpa")
.spi("group").defaultProvider("jpa")
.spi("role").defaultProvider("jpa")
.spi("user").defaultProvider("jpa")
.spi("realm").defaultProvider("jpa")
.spi("serverInfo").defaultProvider("jpa")
;
}
}

6
testsuite/model/src/test/java/org/keycloak/testsuite/model/parameters/JpaFederation.java
View file

@ -31,6 +31,7 @@ import org.keycloak.storage.clientscope.ClientScopeStorageProviderFactory;
import org.keycloak.storage.clientscope.ClientScopeStorageProviderModel;
import org.keycloak.storage.clientscope.ClientScopeStorageProviderSpi;
import org.keycloak.testsuite.federation.HardcodedClientScopeStorageProviderFactory;
import org.keycloak.testsuite.model.Config;
/**
*
@ -70,4 +71,9 @@ public class JpaFederation extends KeycloakModelParameters {
return super.getParameters(clazz);
}
}
@Override
public void updateConfig(Config cf) {
Jpa.updateConfigForJpa(cf);
}
}

7
testsuite/utils/src/main/resources/META-INF/keycloak-server.json
View file

@ -14,6 +14,13 @@
"provider": "${keycloak.eventsStore.provider:}"
},
"serverInfo": {
"provider": "${keycloak.serverInfo.provider:jpa}",
"map": {
"resourcesVersionSeed": "1JZ379bzyOCFA"
}
},
"realm": {
"provider": "${keycloak.realm.provider:jpa}"
},