Run Infinispan using Testcontainers in base testsuite

Closes #13620
This commit is contained in:
Michal Hajas 2022-08-09 14:34:05 +02:00 committed by Hynek Mlnařík
parent 55954b2be7
commit d55d110ff9
13 changed files with 204 additions and 187 deletions

View file

@ -36,7 +36,6 @@ jobs:
./mvnw clean install -nsu -B -e -DskipTests -Pdistribution
./mvnw clean install -nsu -B -e -f testsuite/integration-arquillian/servers/auth-server -Pauth-server-quarkus
./mvnw clean install -nsu -B -e -f testsuite/integration-arquillian/servers/auth-server -Pauth-server-undertow
./mvnw clean install -nsu -B -e -f testsuite/integration-arquillian/servers/cache-server -Pcache-server-infinispan
- name: Store Keycloak artifacts
id: store-keycloak

View file

@ -964,6 +964,90 @@ because this is not UI testing). For debugging purposes you can override the hea
For changing the hostname in the hostname tests (e.g. [DefaultHostnameTest](https://github.com/keycloak/keycloak/blob/main/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/url/DefaultHostnameTest.java)),
we rely on [nip.io](https://nip.io) for DNS switching, so tests will work everywhere without fiddling with `etc/hosts` locally.
## Running base testsuite with Map storage
To run base testsuite with new storage run the following command (this will execute testsuite with ConcurrentHashMap storage):
```shell
mvn clean install -f testsuite/integration-arquillian/tests/base \
-Pauth-server-quarkus -Pmap-storage
```
### Running tests with JPA Map storage
Run PostgreSQL database:
```shell
podman run --name postgres -p 5432:5432 -e POSTGRES_PASSWORD=pass -e POSTGRES_USER=keycloak -e POSTGRES_DB=keycloak -d postgres:13.2
```
Execute tests:
```shell
mvn clean install -f testsuite/integration-arquillian/tests/base \
-Pmap-storage,map-storage-jpa
```
### Running tests with HotRod Map storage
By default, Base testsuite with `map-storage-hotrod` profile spawn a new Infinispan container
with each test execution. To run the tests execute:
```shell
mvn clean install -f testsuite/integration-arquillian/tests/base \
-Pmap-storage,map-storage-hotrod
```
Note: For running Infinispan server we are using Testcontainer, see section
_Usage of Testcontainers_ for details on how to set up your container engine.
It is also possible, to configure Base testsuite to
connect to an external instance of Infinispan. To do so, execute tests with
the following command:
```shell
mvn clean install -f testsuite/integration-arquillian/tests/base \
-Pmap-storage,map-storage-hotrod
-Dkeycloak.testsuite.start-hotrod-container=false \
-Dkeycloak.connectionsHotRod.host=<host> \
-Dkeycloak.connectionsHotRod.port=<port> \
-Dkeycloak.connectionsHotRod.username=<username> \
-Dkeycloak.connectionsHotRod.password=<password>
```
### Usage of Testcontainers
Some profiles within model tests require running 3rd party software, for
example, database or Infinispan. For running these we are using
[Testcontainers](https://www.testcontainers.org/). This may require some
additional configuration of your container engine.
#### Podman settings
For more details see the following [Podman guide from Quarkus webpage](https://quarkus.io/guides/podman).
Specifically, these steps are required:
```shell
# Enable the podman socket with Docker REST API (only needs to be done once)
systemctl --user enable podman.socket --now
# Set the required environment variables (need to be run everytime or added to profile)
export DOCKER_HOST=unix:///run/user/${UID}/podman/podman.sock
```
Testcontainers are using [ryuk](https://hub.docker.com/r/testcontainers/ryuk)
to cleanup containers after tests. To make this work with Podman add the
following line to `~/.testcontainers.properties`
```shell
ryuk.container.privileged=true
```
Alternatively, disable usage of ryuk (using this may result in stale containers
still running after tests finish. This is not recommended especially if you are
executing tests from Intellij IDE as it [may not stop](https://youtrack.jetbrains.com/issue/IDEA-190385)
the containers created during test run).
```shell
export TESTCONTAINERS_RYUK_DISABLED=true #not recommended - see above!
```
#### Docker settings
To use Testcontainers with Docker it is necessary to
[make Docker available for non-root users](https://docs.docker.com/engine/install/linux-postinstall/).
### Tips & Tricks:
Although it _should_ work in general, you may experience an exception like this:
```

View file

@ -255,8 +255,6 @@
<argument>myuser</argument>
<argument>-p</argument>
<argument>"qwer1234!"</argument>
<argument>-g</argument>
<argument>admin</argument>
</arguments>
</configuration>
</execution>

View file

@ -101,7 +101,6 @@
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>

View file

@ -142,10 +142,6 @@ public class AuthServerTestEnricher {
public static final String AUTH_SERVER_CROSS_DC_PROPERTY = "auth.server.crossdc";
public static final boolean AUTH_SERVER_CROSS_DC = Boolean.parseBoolean(System.getProperty(AUTH_SERVER_CROSS_DC_PROPERTY, "false"));
public static final String HOT_ROD_STORE_ENABLED_PROPERTY = "hotrod.store.enabled";
public static final boolean HOT_ROD_STORE_ENABLED = Boolean.parseBoolean(System.getProperty(HOT_ROD_STORE_ENABLED_PROPERTY, "false"));
public static final String AUTH_SERVER_HOME_PROPERTY = "auth.server.home";
public static final String CACHE_SERVER_LIFECYCLE_SKIP_PROPERTY = "cache.server.lifecycle.skip";
@ -350,17 +346,6 @@ public class AuthServerTestEnricher {
}
}
if (HOT_ROD_STORE_ENABLED) {
HotRodStoreTestEnricher.initializeSuiteContext(suiteContext);
for (ContainerInfo container : suiteContext.getContainers()) {
// migrated auth server
if (container.getQualifier().equals("hot-rod-store")) {
suiteContext.setHotRodStoreInfo(container);
}
}
}
suiteContextProducer.set(suiteContext);
CrossDCTestEnricher.initializeSuiteContext(suiteContext);
log.info("\n\n" + suiteContext);

View file

@ -1,53 +1,29 @@
package org.keycloak.testsuite.arquillian;
import org.jboss.arquillian.container.spi.event.StartContainer;
import org.jboss.arquillian.container.spi.event.StopContainer;
import org.jboss.arquillian.container.test.api.ContainerController;
import org.jboss.arquillian.core.api.Event;
import org.jboss.arquillian.core.api.Instance;
import org.jboss.arquillian.core.api.annotation.Inject;
import org.jboss.arquillian.core.api.annotation.Observes;
import org.jboss.arquillian.core.spi.Validate;
import org.jboss.arquillian.test.spi.event.suite.AfterSuite;
import org.jboss.logging.Logger;
import org.jboss.arquillian.container.spi.event.StartSuiteContainers;
import org.jboss.arquillian.core.api.annotation.Observes;
import org.jboss.arquillian.test.spi.event.suite.AfterSuite;
import org.keycloak.testsuite.util.InfinispanContainer;
public class HotRodStoreTestEnricher {
private static SuiteContext suiteContext;
private static final Logger log = Logger.getLogger(HotRodStoreTestEnricher.class);
public static final String HOT_ROD_STORE_HOST_PROPERTY = "keycloak.connectionsHotRod.host";
@Inject
private Event<StartContainer> startContainerEvent;
public static final boolean HOT_ROD_START_CONTAINER = Boolean.parseBoolean(System.getProperty("keycloak.testsuite.start-hotrod-container", "false"));
@Inject
private Event<StopContainer> stopContainerEvent;
static void initializeSuiteContext(SuiteContext suiteContext) {
Validate.notNull(suiteContext, "Suite context cannot be null.");
HotRodStoreTestEnricher.suiteContext = suiteContext;
}
private final InfinispanContainer hotRodContainer = new InfinispanContainer();
public void beforeContainerStarted(@Observes(precedence = 1) StartSuiteContainers event) {
if (!AuthServerTestEnricher.HOT_ROD_STORE_ENABLED) return;
if (!HOT_ROD_START_CONTAINER) return;
hotRodContainer.start();
ContainerInfo hotRodContainer = suiteContext.getHotRodStoreInfo();
if (hotRodContainer != null && !hotRodContainer.isStarted()) {
log.infof("HotRod store starting: %s", hotRodContainer.getQualifier());
startContainerEvent.fire(new StartContainer(hotRodContainer.getArquillianContainer()));
}
// Add env variable, so it can be picked up by Keycloak
System.setProperty(HOT_ROD_STORE_HOST_PROPERTY, hotRodContainer.getHost());
}
public void afterSuite(@Observes(precedence = 4) AfterSuite event) {
if (!AuthServerTestEnricher.HOT_ROD_STORE_ENABLED) return;
ContainerInfo hotRodContainer = suiteContext.getHotRodStoreInfo();
if (hotRodContainer != null && hotRodContainer.isStarted()) {
log.infof("HotRod store stopping: %s", hotRodContainer.getQualifier());
stopContainerEvent.fire(new StopContainer(hotRodContainer.getArquillianContainer()));
}
if (!HOT_ROD_START_CONTAINER) return;
hotRodContainer.stop();
}
}

View file

@ -49,8 +49,6 @@ public final class SuiteContext {
private ContainerInfo migratedAuthServerInfo;
private final MigrationContext migrationContext = new MigrationContext();
private ContainerInfo hotRodStoreInfo;
private boolean adminPasswordUpdated;
private final Map<String, String> smtpServer = new HashMap<>();
@ -176,14 +174,6 @@ public final class SuiteContext {
this.migratedAuthServerInfo = migratedAuthServerInfo;
}
public ContainerInfo getHotRodStoreInfo() {
return hotRodStoreInfo;
}
public void setHotRodStoreInfo(ContainerInfo hotRodStoreInfo) {
this.hotRodStoreInfo = hotRodStoreInfo;
}
public boolean isAuthServerCluster() {
return ! authServerBackendsInfo.isEmpty();
}

View file

@ -0,0 +1,87 @@
/*
* Copyright 2022 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.testsuite.util;
import org.jboss.logging.Logger;
import org.keycloak.testsuite.arquillian.HotRodStoreTestEnricher;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.wait.strategy.Wait;
import java.time.Duration;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class InfinispanContainer extends GenericContainer<InfinispanContainer> {
private final Logger LOG = Logger.getLogger(getClass());
private static final String PORT = System.getProperty("keycloak.connectionsHotRod.port", "11222");
private static String HOST = System.getProperty(HotRodStoreTestEnricher.HOT_ROD_STORE_HOST_PROPERTY);
private static final String USERNAME = System.getProperty("keycloak.connectionsHotRod.username", "admin");
private static final String PASSWORD = System.getProperty("keycloak.connectionsHotRod.password", "admin");
private static final String ZERO_TO_255
= "(\\d{1,2}|(0|1)\\"
+ "d{2}|2[0-4]\\d|25[0-5])";
private static final String IP_ADDRESS_REGEX
= ZERO_TO_255 + "\\."
+ ZERO_TO_255 + "\\."
+ ZERO_TO_255 + "\\."
+ ZERO_TO_255;
private static final Pattern IP_ADDRESS_PATTERN = Pattern.compile("listening on (" + IP_ADDRESS_REGEX + "):" + PORT);
public InfinispanContainer() {
super("quay.io/infinispan/server:" + System.getProperty("infinispan.version"));
withEnv("USER", USERNAME);
withEnv("PASS", PASSWORD);
withNetworkMode("host");
withStartupTimeout(Duration.ofMinutes(5));
waitingFor(Wait.forLogMessage(".*Infinispan Server.*started in.*", 1));
}
public String getHost() {
if (HOST == null && this.isRunning()) {
Matcher matcher = IP_ADDRESS_PATTERN.matcher(getLogs());
if (!matcher.find()) {
LOG.errorf("Cannot find IP address of the infinispan server in log:\\n%s ", getLogs());
throw new IllegalStateException("Cannot find IP address of the Infinispan server. See test log for Infinispan container log.");
}
HOST = matcher.group(1);
}
return HOST;
}
@Override
public void start() {
super.start();
}
public String getPort() {
return PORT;
}
public String getUsername() {
return USERNAME;
}
public String getPassword() {
return PASSWORD;
}
}

View file

@ -297,13 +297,12 @@
"connectionsHotRod": {
"default": {
"embedded": "${keycloak.connectionsHotRod.embedded:false}",
"port": "${keycloak.connectionsHotRod.port:14232}",
"port": "${keycloak.connectionsHotRod.port:11222}",
"host": "${keycloak.connectionsHotRod.host:127.0.0.1}",
"configureRemoteCaches": "${keycloak.connectionsHotRod.configureRemoteCaches:true}",
"username": "${keycloak.connectionsHotRod.username:myuser}",
"password": "${keycloak.connectionsHotRod.password:qwer1234!}",
"enableSecurity": "${keycloak.connectionsHotRod.enableSecurity:true}",
"reindexCaches": "${keycloak.connectionsHotRod.reindexCaches:all}"
"username": "${keycloak.connectionsHotRod.username:admin}",
"password": "${keycloak.connectionsHotRod.password:admin}",
"enableSecurity": "${keycloak.connectionsHotRod.enableSecurity:true}"
}
},

View file

@ -641,19 +641,6 @@
</container>
</group>
<container qualifier="hot-rod-store" mode="manual" >
<configuration>
<property name="enabled">${hotrod.store.enabled}</property>
<property name="adapterImplClass">org.keycloak.testsuite.arquillian.containers.InfinispanServerDeployableContainer</property>
<property name="infinispanHome">${cache.server.home}-hot-rod-store</property>
<!-- <property name="serverConfig">infinisan-xsite.xml</property>-->
<property name="portOffset">${hotrod.store.port.offset}</property>
<property name="managementPort">${hotrod.store.management.port}</property>
<property name="javaHome">${cache.server.java.home}</property>
</configuration>
</container>
<container qualifier="auth-server-remote" mode="manual" >
<configuration>
<property name="enabled">${auth.server.remote}</property>

View file

@ -49,7 +49,6 @@
<auth.server.quarkus.cluster>false</auth.server.quarkus.cluster>
<auth.server.crossdc>false</auth.server.crossdc>
<hotrod.store.enabled>false</hotrod.store.enabled>
<auth.server.undertow.crossdc>false</auth.server.undertow.crossdc>
<auth.server.jboss.crossdc>false</auth.server.jboss.crossdc>
<cache.server.lifecycle.skip>false</cache.server.lifecycle.skip>
@ -141,9 +140,6 @@
<cache.server.console.output>true</cache.server.console.output>
<cache.server.auth>false</cache.server.auth>
<hotrod.store.port.offset>3010</hotrod.store.port.offset>
<hotrod.store.management.port>13000</hotrod.store.management.port>
<!--
~ Definition of default JVM parameters for all modular JDKs. See:
~
@ -670,11 +666,6 @@
<auth.server.jboss.crossdc>${auth.server.jboss.crossdc}</auth.server.jboss.crossdc>
<cache.server.lifecycle.skip>${cache.server.lifecycle.skip}</cache.server.lifecycle.skip>
<!--hot-rod-store properties-->
<hotrod.store.enabled>${hotrod.store.enabled}</hotrod.store.enabled>
<hotrod.store.port.offset>${hotrod.store.port.offset}</hotrod.store.port.offset>
<hotrod.store.management.port>${hotrod.store.management.port}</hotrod.store.management.port>
<cache.server>${cache.server}</cache.server>
<cache.server.legacy>${cache.server.legacy}</cache.server.legacy>
<cache.server.1.port.offset>${cache.server.1.port.offset}</cache.server.1.port.offset>
@ -1434,9 +1425,7 @@
<profile>
<id>map-storage-hot-rod</id>
<properties>
<hotrod.store.enabled>true</hotrod.store.enabled>
<skip.copy.hotrod.server>false</skip.copy.hotrod.server>
<cache.server>infinispan</cache.server>
<keycloak.testsuite.start-hotrod-container>true</keycloak.testsuite.start-hotrod-container>
<auth.server.quarkus.mapStorage.profile.config>hotrod</auth.server.quarkus.mapStorage.profile.config>
</properties>
<build>
@ -1444,54 +1433,6 @@
</plugins>
<pluginManagement>
<plugins>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>unpack-cache-server-standalone-infinispan</id>
<phase>generate-resources</phase>
<goals>
<goal>unpack</goal>
</goals>
<configuration>
<artifactItems>
<artifactItem>
<groupId>org.keycloak.testsuite</groupId>
<artifactId>integration-arquillian-servers-cache-server-infinispan-infinispan</artifactId>
<version>${project.version}</version>
<type>zip</type>
<outputDirectory>${containers.home}</outputDirectory>
</artifactItem>
</artifactItems>
<overWriteIfNewer>true</overWriteIfNewer>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<id>copy-cache-server-to-hot-rod-directory</id>
<phase>process-resources</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<skip>${skip.copy.hotrod.server}</skip>
<target>
<move todir="${cache.server.home}-hot-rod-store">
<fileset dir="${cache.server.home}"/>
</move>
<chmod dir="${cache.server.home}-hot-rod-store/bin" perm="ugo+rx" includes="**/*.sh"/>
</target>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
@ -1510,6 +1451,8 @@
<keycloak.eventStore.map.storage.provider>hotrod</keycloak.eventStore.map.storage.provider>
<keycloak.actionToken.map.storage.provider>hotrod</keycloak.actionToken.map.storage.provider>
<keycloak.singleUseObject.map.storage.provider>hotrod</keycloak.singleUseObject.map.storage.provider>
<infinispan.version>${infinispan.version}</infinispan.version>
<keycloak.testsuite.start-hotrod-container>${keycloak.testsuite.start-hotrod-container}</keycloak.testsuite.start-hotrod-container>
</systemPropertyVariables>
</configuration>
</plugin>

View file

@ -98,9 +98,9 @@ connect to an external instance of Infinispan. To do so, execute tests with
the following command:
```shell
mvn test -Phot-rod \
-Dhot-rod.start-container=false \
-Dhot-rod.connection.host=<host> \
-Dhot-rod.connection.port=<port> \
-Dhot-rod.connection.username=<username> \
-Dhot-rod.connection.password=<password>
-Dkeycloak.testsuite.start-hotrod-container=false \
-Dkeycloak.connectionsHotRod.host=<host> \
-Dkeycloak.connectionsHotRod.port=<port> \
-Dkeycloak.connectionsHotRod.username=<username> \
-Dkeycloak.connectionsHotRod.password=<password>
```

View file

@ -50,6 +50,7 @@ import org.keycloak.provider.Spi;
import org.keycloak.sessions.AuthenticationSessionSpi;
import org.keycloak.testsuite.model.Config;
import org.keycloak.testsuite.model.KeycloakModelParameters;
import org.keycloak.testsuite.util.InfinispanContainer;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.wait.strategy.Wait;
@ -65,23 +66,7 @@ import java.util.regex.Pattern;
public class HotRodMapStorage extends KeycloakModelParameters {
private final Logger LOG = Logger.getLogger(getClass());
public static final String PORT = System.getProperty("hot-rod.connection.port", "11222");
public static String HOST = System.getProperty("hot-rod.connection.host");
public static final String USERNAME = System.getProperty("hot-rod.connection.username", "admin");
public static final String PASSWORD = System.getProperty("hot-rod.connection.password", "admin");
public static final Boolean START_CONTAINER = Boolean.valueOf(System.getProperty("hot-rod.start-container", "true"));
private static final String ZERO_TO_255
= "(\\d{1,2}|(0|1)\\"
+ "d{2}|2[0-4]\\d|25[0-5])";
private static final String IP_ADDRESS_REGEX
= ZERO_TO_255 + "\\."
+ ZERO_TO_255 + "\\."
+ ZERO_TO_255 + "\\."
+ ZERO_TO_255;
private static final Pattern IP_ADDRESS_PATTERN = Pattern.compile("listening on (" + IP_ADDRESS_REGEX + "):" + PORT);
public static final Boolean START_CONTAINER = Boolean.valueOf(System.getProperty("keycloak.testsuite.start-hotrod-container", "true"));
static final Set<Class<? extends Spi>> ALLOWED_SPIS = ImmutableSet.<Class<? extends Spi>>builder()
.add(HotRodConnectionSpi.class)
.build();
@ -91,10 +76,7 @@ public class HotRodMapStorage extends KeycloakModelParameters {
.add(HotRodConnectionProviderFactory.class)
.build();
private final GenericContainer<?> hotRodContainer = new GenericContainer("quay.io/infinispan/server:" + System.getProperty("infinispan.version"))
.withEnv("USER", USERNAME)
.withEnv("PASS", PASSWORD)
.withNetworkMode("host");
private final InfinispanContainer hotRodContainer = new InfinispanContainer();
@Override
public void updateConfig(Config cf) {
@ -121,30 +103,18 @@ public class HotRodMapStorage extends KeycloakModelParameters {
.config("dir", "${project.build.directory:target}")
.config("keyType.single-use-objects", "string");
if (HOST == null && START_CONTAINER) {
Matcher matcher = IP_ADDRESS_PATTERN.matcher(hotRodContainer.getLogs());
if (!matcher.find()) {
LOG.errorf("Cannot find IP address of the infinispan server in log:\\n%s ", hotRodContainer.getLogs());
throw new IllegalStateException("Cannot find IP address of the Infinispan server. See test log for Infinispan container log.");
}
HOST = matcher.group(1);
}
cf.spi(HotRodConnectionSpi.NAME).provider(DefaultHotRodConnectionProviderFactory.PROVIDER_ID)
.config("host", HOST)
.config("port", PORT)
.config("username", USERNAME)
.config("password", PASSWORD)
.config("host", hotRodContainer.getHost())
.config("port", hotRodContainer.getPort())
.config("username", hotRodContainer.getUsername())
.config("password", hotRodContainer.getPassword())
.config("configureRemoteCaches", "true");
}
@Override
public void beforeSuite(Config cf) {
if (START_CONTAINER) {
hotRodContainer
.withStartupTimeout(Duration.ofMinutes(5))
.waitingFor(Wait.forLogMessage(".*Infinispan Server.*started in.*", 1))
.start();
hotRodContainer.start();
}
}