libre.sh/keycloak-scim
3
0
Fork 0
Code Issues 15 Pull requests Releases 1 Packages Activity Actions

Adding CRDB into GHA for the new store (#16021)

Browse source 
The CockroachDB database is slower than PostgreSQL, therefore it will only run branches and nightly builds.

Closes #16020
This commit is contained in:
Alexander Schwartz 2022-12-17 08:50:21 +01:00 committed by GitHub
parent e1a8a30696
commit 1d758fac2b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 298 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
.github/workflows/ci.yml vendored
View file

@ -165,7 +165,7 @@ jobs:
run: |
TESTS=`testsuite/integration-arquillian/tests/base/testsuites/suite.sh jdk`
echo "Tests: $TESTS"
./mvnw test -nsu -B -Pauth-server-quarkus -Pdb-${{ matrix.db }} -Dtest=$TESTS -pl testsuite/integration-arquillian/tests/base | misc/log/trimmer.sh
./mvnw test -nsu -B -Pauth-server-quarkus -Dtest=$TESTS -pl testsuite/integration-arquillian/tests/base | misc/log/trimmer.sh
- id: upload-surefire-reports
name: Upload Surefire reports
@ -175,14 +175,31 @@ jobs:
api-key: ${{ secrets.FORESIGHT_API_KEY }}
surefire-reports-path: 'testsuite/integration-arquillian/tests/base/target/surefire-reports/*.xml'
databases-new-store:
name: Databases New Store
runs-on: ubuntu-latest
steps:
- id: matrix-db
# language=bash
run: |
# For PRs, run only PostgreSQL, on branches and nightly runs run all databases
if [ "${{ github.event_name }}" == "pull_request" ]; then
echo 'db=["chm", "hot-rod", "jpa-postgres"]' >> $GITHUB_OUTPUT
else
echo 'db=["chm", "hot-rod", "jpa-postgres", "jpa-cockroach"]' >> $GITHUB_OUTPUT
fi
outputs:
db: ${{ steps.matrix-db.outputs.db }}
new-store-integration-tests:
name: New Store IT
needs: build
needs: [build, databases-new-store]
runs-on: ubuntu-latest
timeout-minutes: 45
# Currently, the run of CockroachDB (only branches and nightly runs) is the longest with ~45 minutes
timeout-minutes: 70
strategy:
matrix:
db: [chm, hot-rod, jpa]
db: ${{ fromJson(needs.databases-new-store.outputs.db) }}
fail-fast: false
steps:
- uses: actions/checkout@v3
@ -196,7 +213,8 @@ jobs:
declare -A PARAMS
PARAMS["chm"]="-Pmap-storage -Dpageload.timeout=90000"
PARAMS["hot-rod"]="-Pmap-storage,map-storage-hot-rod -Dpageload.timeout=90000"
PARAMS["jpa"]="-Pmap-storage,map-storage-jpa -Dpageload.timeout=90000"
PARAMS["jpa-postgres"]="-Pmap-storage,map-storage-jpa-postgres -Dpageload.timeout=90000"
PARAMS["jpa-cockroach"]="-Pmap-storage,map-storage-jpa-cockroach -Dpageload.timeout=90000"
TESTS=`testsuite/integration-arquillian/tests/base/testsuites/suite.sh database`
echo "Tests: $TESTS"

4
quarkus/tests/integration/pom.xml
View file

@ -88,6 +88,10 @@
<groupId>org.testcontainers</groupId>
<artifactId>postgresql</artifactId>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>cockroachdb</artifactId>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>mariadb</artifactId>

15
testsuite/integration-arquillian/HOW-TO-RUN.md
View file

@ -837,14 +837,19 @@ mvn clean install -f testsuite/integration-arquillian/tests/base \
### Running tests with JPA Map storage
By default tests with `map-storage-jpa` profile spawns a new Postgres container
with each test execution. Default image used is "postgres:alpine". To spawn different
version, it can be used "keycloak.map.storage.postgres.docker.image" system property.
By default, testing with the profile `map-storage-jpa-postgres` spawns a new Postgres container
with each test execution. The default image used is `postgres:alpine`. To spawn a different
version, use the system property `keycloak.map.storage.postgres.docker.image`.
In a similar way the profile `map-storage-jpa-cockroach` spawns a new CockroachDB container
with each test execution. It uses the official CockroachDB image in the version stated in the
class `CockroachdbContainerTestEnricher`. To spawn a different
version, use the system property `keycloak.map.storage.cockroachdb.docker.image`.
Execute tests:
```shell
mvn clean install -f testsuite/integration-arquillian/tests/base \
-Pmap-storage,map-storage-jpa
-Pmap-storage,map-storage-jpa-postgres
```
It's also possible to configure tests to connect to an external database, it might be useful
@ -859,7 +864,7 @@ podman run --name postgres -p 5432:5432 -e POSTGRES_PASSWORD=pass -e POSTGRES_US
To run the tests without spawning the container for you, execute tests with the following command:
```shell
mvn clean install -f testsuite/integration-arquillian/tests/base \
-Pmap-storage,map-storage-jpa \
-Pmap-storage,map-storage-jpa-postgres \
-Dpostgres.start-container=false \
-Dkeycloak.map.storage.connectionsJpa.url=<jdbc_url> \
-Dkeycloak.map.storage.connectionsJpa.user=<user> \

62
testsuite/integration-arquillian/tests/base/pom.xml
View file

@ -117,6 +117,11 @@
<artifactId>postgresql</artifactId>
<version>${testcontainers.version}</version>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>cockroachdb</artifactId>
<version>${testcontainers.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
@ -814,7 +819,7 @@
</profile>
<profile>
<id>map-storage-jpa</id>
<id>map-storage-jpa-postgres</id>
<properties>
<postgres.start-container>true</postgres.start-container>
</properties>
@ -866,6 +871,61 @@
</build>
</profile>
<profile>
<id>map-storage-jpa-cockroach</id>
<properties>
<cockroachdb.start-container>true</cockroachdb.start-container>
<keycloak.map.storage.connectionsJpa.url>jdbc:postgresql://localhost:26257/keycloak</keycloak.map.storage.connectionsJpa.url>
</properties>
<build>
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<goals>
<goal>enforce</goal>
</goals>
<configuration>
<rules>
<requireActiveProfile>
<profiles>map-storage</profiles>
<message>map-storage profile not active</message>
</requireActiveProfile>
</rules>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<systemPropertyVariables>
<cockroachdb.start-container>${cockroachdb.start-container}</cockroachdb.start-container>
<keycloak.map.storage.connectionsJpa.url>${keycloak.map.storage.connectionsJpa.url}</keycloak.map.storage.connectionsJpa.url>
<keycloak.map.storage.connectionsJpa.user>${keycloak.map.storage.connectionsJpa.user}</keycloak.map.storage.connectionsJpa.user>
<keycloak.map.storage.connectionsJpa.password>${keycloak.map.storage.connectionsJpa.password}</keycloak.map.storage.connectionsJpa.password>
<keycloak.authorization.map.storage.provider>jpa</keycloak.authorization.map.storage.provider>
<keycloak.authSession.map.storage.provider>jpa</keycloak.authSession.map.storage.provider>
<keycloak.client.map.storage.provider>jpa</keycloak.client.map.storage.provider>
<keycloak.clientScope.map.storage.provider>jpa</keycloak.clientScope.map.storage.provider>
<keycloak.adminEventsStore.map.storage.provider>jpa</keycloak.adminEventsStore.map.storage.provider>
<keycloak.authEventsStore.map.storage.provider>jpa</keycloak.authEventsStore.map.storage.provider>
<keycloak.group.map.storage.provider>jpa</keycloak.group.map.storage.provider>
<keycloak.loginFailure.map.storage.provider>jpa</keycloak.loginFailure.map.storage.provider>
<keycloak.realm.map.storage.provider>jpa</keycloak.realm.map.storage.provider>
<keycloak.role.map.storage.provider>jpa</keycloak.role.map.storage.provider>
<keycloak.singleUseObject.map.storage.provider>jpa</keycloak.singleUseObject.map.storage.provider>
<keycloak.user.map.storage.provider>jpa</keycloak.user.map.storage.provider>
<keycloak.userSession.map.storage.provider>jpa</keycloak.userSession.map.storage.provider>
<auth.server.quarkus.mapStorage.profile.config>jpa</auth.server.quarkus.mapStorage.profile.config>
</systemPropertyVariables>
</configuration>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>auth-server-fips140-2</id>

52
testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/CockroachdbContainerTestEnricher.java Normal file
View file

@ -0,0 +1,52 @@
/*
* Copyright 2022 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.testsuite.arquillian;
import org.jboss.arquillian.container.spi.event.StartSuiteContainers;
import org.jboss.arquillian.core.api.annotation.Observes;
import org.jboss.arquillian.test.spi.event.suite.AfterSuite;
import org.testcontainers.containers.CockroachContainer;
import org.testcontainers.utility.DockerImageName;
public class CockroachdbContainerTestEnricher {
private static final Boolean START_CONTAINER = Boolean.valueOf(System.getProperty("cockroachdb.start-container", "false"));
private static final String COCKROACHDB_DOCKER_IMAGE_NAME = System.getProperty("keycloak.map.storage.cockroachdb.docker.image", "cockroachdb/cockroach:v22.1.0");
private static final CockroachContainer COCKROACHDB_CONTAINER = new CockroachContainer(DockerImageName.parse(COCKROACHDB_DOCKER_IMAGE_NAME).asCompatibleSubstituteFor("cockroachdb"));
private static final String COCKROACHDB_DB_USER = System.getProperty("keycloak.map.storage.connectionsJpa.user", "keycloak");
public void beforeContainerStarted(@Observes(precedence = 1) StartSuiteContainers event) {
if (START_CONTAINER) {
COCKROACHDB_CONTAINER
// Using the environment variables for now where using the withXXX() method is not supported, yet.
// https://github.com/testcontainers/testcontainers-java/issues/6299
.withEnv("COCKROACH_DATABASE", "keycloak")
.withEnv("COCKROACH_USER", COCKROACHDB_DB_USER)
// password is not used/supported in insecure mode
.withCommand("start-single-node", "--insecure")
.start();
System.setProperty("keycloak.map.storage.connectionsJpa.url", COCKROACHDB_CONTAINER.getJdbcUrl());
}
}
public void afterSuite(@Observes(precedence = 4) AfterSuite event) {
if (START_CONTAINER) {
COCKROACHDB_CONTAINER.stop();
}
}
}

1
testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/KeycloakArquillianExtension.java
View file

@ -68,6 +68,7 @@ public class KeycloakArquillianExtension implements LoadableExtension {
.observer(CrossDCTestEnricher.class)
.observer(HotRodStoreTestEnricher.class)
.observer(PostgresContainerTestEnricher.class)
.observer(CockroachdbContainerTestEnricher.class)
.observer(H2TestEnricher.class);
builder
.service(TestExecutionDecider.class, MigrationTestExecutionDecider.class)

8
testsuite/model/pom.xml
View file

@ -344,6 +344,14 @@
</properties>
</profile>
<profile>
<id>map-jpa-cockroachdb</id>
<properties>
<keycloak.profile.feature.map_storage>enabled</keycloak.profile.feature.map_storage>
<keycloak.model.parameters>Map,JpaMapStorageCockroachdb</keycloak.model.parameters>
</properties>
</profile>
<profile>
<id>.asyncProfiler</id>
<activation>

139
testsuite/model/src/test/java/org/keycloak/testsuite/model/parameters/JpaMapStorageCockroachdb.java Normal file
View file

@ -0,0 +1,139 @@
/*
* Copyright 2022 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.testsuite.model.parameters;
import com.google.common.collect.ImmutableSet;
import org.jboss.logging.Logger;
import org.keycloak.authorization.store.StoreFactorySpi;
import org.keycloak.events.EventStoreSpi;
import org.keycloak.models.DeploymentStateSpi;
import org.keycloak.models.SingleUseObjectSpi;
import org.keycloak.models.UserLoginFailureSpi;
import org.keycloak.models.UserSessionSpi;
import org.keycloak.models.map.authSession.MapRootAuthenticationSessionProviderFactory;
import org.keycloak.models.map.authorization.MapAuthorizationStoreFactory;
import org.keycloak.models.map.client.MapClientProviderFactory;
import org.keycloak.models.map.clientscope.MapClientScopeProviderFactory;
import org.keycloak.models.map.deploymentState.MapDeploymentStateProviderFactory;
import org.keycloak.models.map.events.MapEventStoreProviderFactory;
import org.keycloak.models.map.group.MapGroupProviderFactory;
import org.keycloak.models.map.keys.MapPublicKeyStorageProviderFactory;
import org.keycloak.models.map.loginFailure.MapUserLoginFailureProviderFactory;
import org.keycloak.models.map.realm.MapRealmProviderFactory;
import org.keycloak.models.map.role.MapRoleProviderFactory;
import org.keycloak.models.map.singleUseObject.MapSingleUseObjectProviderFactory;
import org.keycloak.models.map.storage.MapStorageSpi;
import org.keycloak.models.map.storage.chm.ConcurrentHashMapStorageProviderFactory;
import org.keycloak.models.map.storage.jpa.JpaMapStorageProviderFactory;
import org.keycloak.models.map.storage.jpa.liquibase.connection.MapLiquibaseConnectionProviderFactory;
import org.keycloak.models.map.storage.jpa.liquibase.connection.MapLiquibaseConnectionSpi;
import org.keycloak.models.map.storage.jpa.updater.MapJpaUpdaterProviderFactory;
import org.keycloak.models.map.storage.jpa.updater.MapJpaUpdaterSpi;
import org.keycloak.models.map.user.MapUserProviderFactory;
import org.keycloak.models.map.userSession.MapUserSessionProviderFactory;
import org.keycloak.provider.ProviderFactory;
import org.keycloak.provider.Spi;
import org.keycloak.sessions.AuthenticationSessionSpi;
import org.keycloak.testsuite.model.Config;
import org.keycloak.testsuite.model.KeycloakModelParameters;
import org.testcontainers.containers.CockroachContainer;
import org.testcontainers.utility.DockerImageName;
import java.util.Set;
public class JpaMapStorageCockroachdb extends KeycloakModelParameters {
private static final Logger LOG = Logger.getLogger(JpaMapStorageCockroachdb.class.getName());
private static final Boolean START_CONTAINER = Boolean.valueOf(System.getProperty("cockroachdb.start-container", "true"));
private static final String COCKROACHDB_DOCKER_IMAGE_NAME = System.getProperty("keycloak.map.storage.cockroachdb.docker.image", "cockroachdb/cockroach:v22.1.0");
private static final CockroachContainer COCKROACHDB_CONTAINER = new CockroachContainer(DockerImageName.parse(COCKROACHDB_DOCKER_IMAGE_NAME).asCompatibleSubstituteFor("cockroachdb"));
private static final String COCKROACHDB_DB_USER = System.getProperty("keycloak.map.storage.connectionsJpa.user", "keycloak");
private static final String COCKROACHDB_DB_PASSWORD = System.getProperty("keycloak.map.storage.connectionsJpa.password", "pass");
private static String COCKROACHDB_DB_JDBC_URL = System.getProperty("keycloak.map.storage.connectionsJpa.url");
static final Set<Class<? extends Spi>> ALLOWED_SPIS = ImmutableSet.<Class<? extends Spi>>builder()
.add(MapJpaUpdaterSpi.class)
.add(MapLiquibaseConnectionSpi.class)
.build();
static final Set<Class<? extends ProviderFactory>> ALLOWED_FACTORIES = ImmutableSet.<Class<? extends ProviderFactory>>builder()
.add(ConcurrentHashMapStorageProviderFactory.class)
.add(JpaMapStorageProviderFactory.class)
.add(MapJpaUpdaterProviderFactory.class)
.add(MapLiquibaseConnectionProviderFactory.class)
.build();
public JpaMapStorageCockroachdb() {
super(ALLOWED_SPIS, ALLOWED_FACTORIES);
}
@Override
public void updateConfig(Config cf) {
cf.spi(MapStorageSpi.NAME)
.provider(ConcurrentHashMapStorageProviderFactory.PROVIDER_ID)
.config("dir", "${project.build.directory:target}");
cf.spi(MapStorageSpi.NAME)
.provider(JpaMapStorageProviderFactory.PROVIDER_ID)
.config("url", COCKROACHDB_DB_JDBC_URL)
.config("user", COCKROACHDB_DB_USER)
.config("password", COCKROACHDB_DB_PASSWORD)
.config("driver", "org.postgresql.Driver")
.config("driverDialect", "org.keycloak.models.map.storage.jpa.hibernate.dialect.JsonbPostgreSQL95Dialect");
cf.spi(AuthenticationSessionSpi.PROVIDER_ID).provider(MapRootAuthenticationSessionProviderFactory.PROVIDER_ID) .config(STORAGE_CONFIG, JpaMapStorageProviderFactory.PROVIDER_ID)
.spi("client").provider(MapClientProviderFactory.PROVIDER_ID) .config(STORAGE_CONFIG, JpaMapStorageProviderFactory.PROVIDER_ID)
.spi("clientScope").provider(MapClientScopeProviderFactory.PROVIDER_ID) .config(STORAGE_CONFIG, JpaMapStorageProviderFactory.PROVIDER_ID)
.spi("group").provider(MapGroupProviderFactory.PROVIDER_ID) .config(STORAGE_CONFIG, JpaMapStorageProviderFactory.PROVIDER_ID)
.spi("realm").provider(MapRealmProviderFactory.PROVIDER_ID) .config(STORAGE_CONFIG, JpaMapStorageProviderFactory.PROVIDER_ID)
.spi("role").provider(MapRoleProviderFactory.PROVIDER_ID) .config(STORAGE_CONFIG, JpaMapStorageProviderFactory.PROVIDER_ID)
.spi(DeploymentStateSpi.NAME).provider(MapDeploymentStateProviderFactory.PROVIDER_ID) .config(STORAGE_CONFIG, ConcurrentHashMapStorageProviderFactory.PROVIDER_ID)
.spi(StoreFactorySpi.NAME).provider(MapAuthorizationStoreFactory.PROVIDER_ID) .config(STORAGE_CONFIG, JpaMapStorageProviderFactory.PROVIDER_ID)
.spi("user").provider(MapUserProviderFactory.PROVIDER_ID) .config(STORAGE_CONFIG, JpaMapStorageProviderFactory.PROVIDER_ID)
.spi(UserLoginFailureSpi.NAME).provider(MapUserLoginFailureProviderFactory.PROVIDER_ID) .config(STORAGE_CONFIG, JpaMapStorageProviderFactory.PROVIDER_ID)
.spi(SingleUseObjectSpi.NAME).provider(MapSingleUseObjectProviderFactory.PROVIDER_ID) .config(STORAGE_CONFIG, JpaMapStorageProviderFactory.PROVIDER_ID)
.spi("publicKeyStorage").provider(MapPublicKeyStorageProviderFactory.PROVIDER_ID) .config(STORAGE_CONFIG, ConcurrentHashMapStorageProviderFactory.PROVIDER_ID)
.spi(UserSessionSpi.NAME).provider(MapUserSessionProviderFactory.PROVIDER_ID) .config(STORAGE_CONFIG, JpaMapStorageProviderFactory.PROVIDER_ID)
.spi(EventStoreSpi.NAME).provider(MapEventStoreProviderFactory.PROVIDER_ID) .config("storage-admin-events.provider", JpaMapStorageProviderFactory.PROVIDER_ID)
.config("storage-auth-events.provider", JpaMapStorageProviderFactory.PROVIDER_ID);
}
@Override
public void beforeSuite(Config cf) {
if (START_CONTAINER) {
COCKROACHDB_CONTAINER
// Using the environment variables for now where using the withXXX() method is not supported, yet.
// https://github.com/testcontainers/testcontainers-java/issues/6299
.withEnv("COCKROACH_DATABASE", "keycloak")
.withEnv("COCKROACH_USER", COCKROACHDB_DB_USER)
// password is not used/supported in insecure mode
.withCommand("start-single-node", "--insecure")
.start();
COCKROACHDB_DB_JDBC_URL = COCKROACHDB_CONTAINER.getJdbcUrl();
}
}
@Override
public void afterSuite() {
if (START_CONTAINER) {
COCKROACHDB_CONTAINER.stop();
}
}
}