libre.sh/keycloak-scim
3
0
Fork 0
Code Issues 15 Pull requests Releases Packages Activity Actions

Operator Clustering support

Browse source 
Co-authored-by: Jonathan Vila <jvilalop@redhat.com>
Co-authored-by: Andrea Peruffo <andrea.peruffo1982@gmail.com>
This commit is contained in:
Jonathan Vila 2022-02-08 15:13:58 +01:00 committed by Pedro Igor
parent 92f6c75328
commit c4b978b6c8
20 changed files with 2373 additions and 132 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/operator-ci.yml vendored
View file

@ -61,7 +61,7 @@ jobs:
- name: Test operator running in cluster
working-directory: operator
run: |
eval $(minikube -p minikube docker-env)
eval $(minikube -p minikube docker-env)
mvn clean verify \
-Dquarkus.container-image.build=true -Dquarkus.container-image.tag=test \
-Dquarkus.kubernetes.deployment-target=kubernetes \

16
operator/README.md
View file

@ -64,3 +64,19 @@ Remove the created resources with:
```bash
kubectl delete -k <previously-used-folder>
```
### Testing
Testing allows 2 methods specified in the property `test.operator.deployment` : `local` & `remote`.
`local` : resources will be deployed to the local cluster and the operator will run out of the cluster
`remote` : same as local test but an image for the operator will be generated and deployed run inside the cluster
```bash
mvn clean verify \
-Dquarkus.container-image.build=true \
-Dquarkus.container-image.tag=test \
-Dquarkus.kubernetes.deployment-target=kubernetes \
-Dtest.operator.deployment=remote
```

5
operator/pom.xml
View file

@ -137,6 +137,11 @@
<version>${awaitility.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.rest-assured</groupId>
<artifactId>rest-assured</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>

10
operator/src/main/java/org/keycloak/operator/Constants.java
View file

@ -33,7 +33,9 @@ public final class Constants {
);
public static final Map<String, String> DEFAULT_DIST_CONFIG = Map.of(
"KC_HEALTH_ENABLED", "true"
"KC_HEALTH_ENABLED","true",
"KC_CACHE", "ispn",
"KC_CACHE_STACK", "kubernetes"
);
// Init container
@ -42,4 +44,10 @@ public final class Constants {
public static final String INIT_CONTAINER_NAME = "keycloak-extensions";
public static final String INIT_CONTAINER_EXTENSIONS_FOLDER = "/opt/extensions";
public static final String INIT_CONTAINER_EXTENSIONS_ENV_VAR = "KEYCLOAK_EXTENSIONS";
public static final Integer KEYCLOAK_SERVICE_PORT = 8080;
public static final String KEYCLOAK_SERVICE_PROTOCOL = "TCP";
public static final String KEYCLOAK_SERVICE_SUFFIX = "-service";
public static final Integer KEYCLOAK_DISCOVERY_SERVICE_PORT = 7800;
public static final String KEYCLOAK_DISCOVERY_SERVICE_SUFFIX = "-discovery";
}

6
operator/src/main/java/org/keycloak/operator/OperatorManagedResource.java
View file

@ -85,4 +85,10 @@ public abstract class OperatorManagedResource {
resource.getMetadata().setOwnerReferences(Collections.singletonList(owner));
}
protected String getNamespace() {
return cr.getMetadata().getNamespace();
}
protected abstract String getName();
}

6
operator/src/main/java/org/keycloak/operator/StatusUpdater.java Normal file
View file

@ -0,0 +1,6 @@
package org.keycloak.operator;
public interface StatusUpdater<T> {
void updateStatus(T status);
}

22
operator/src/main/java/org/keycloak/operator/v2alpha1/KeycloakController.java
View file

@ -16,8 +16,7 @@
*/
package org.keycloak.operator.v2alpha1;
import javax.inject.Inject;
import io.fabric8.kubernetes.api.model.Service;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.informers.SharedIndexInformer;
@ -39,7 +38,7 @@ import org.keycloak.operator.v2alpha1.crds.Keycloak;
import org.keycloak.operator.v2alpha1.crds.KeycloakStatus;
import org.keycloak.operator.v2alpha1.crds.KeycloakStatusBuilder;
import java.util.Collections;
import javax.inject.Inject;
import java.util.List;
import java.util.Optional;
@ -62,9 +61,15 @@ public class KeycloakController implements Reconciler<Keycloak>, EventSourceInit
.withLabels(Constants.DEFAULT_LABELS)
.runnableInformer(0);
EventSource deploymentEvent = new InformerEventSource<>(deploymentInformer, Mappers.fromOwnerReference());
SharedIndexInformer<Service> servicesInformer =
client.services().inNamespace(context.getConfigurationService().getClientConfiguration().getNamespace())
.withLabels(Constants.DEFAULT_LABELS)
.runnableInformer(0);
return List.of(deploymentEvent);
EventSource deploymentEvent = new InformerEventSource<>(deploymentInformer, Mappers.fromOwnerReference());
EventSource servicesEvent = new InformerEventSource<>(servicesInformer, Mappers.fromOwnerReference());
return List.of(deploymentEvent, servicesEvent);
}
@Override
@ -82,6 +87,13 @@ public class KeycloakController implements Reconciler<Keycloak>, EventSourceInit
kcDeployment.updateStatus(statusBuilder);
kcDeployment.createOrUpdateReconciled();
var kcService = new KeycloakService(client, kc);
kcService.updateStatus(statusBuilder);
kcService.createOrUpdateReconciled();
var kcDiscoveryService = new KeycloakDiscoveryService(client, kc);
kcDiscoveryService.updateStatus(statusBuilder);
kcDiscoveryService.createOrUpdateReconciled();
var status = statusBuilder.build();
Log.info("--- Reconciliation finished successfully");

33
operator/src/main/java/org/keycloak/operator/v2alpha1/KeycloakDeployment.java
View file

@ -18,6 +18,7 @@ package org.keycloak.operator.v2alpha1;
import io.fabric8.kubernetes.api.model.Container;
import io.fabric8.kubernetes.api.model.ContainerBuilder;
import io.fabric8.kubernetes.api.model.EnvVar;
import io.fabric8.kubernetes.api.model.EnvVarBuilder;
import io.fabric8.kubernetes.api.model.HasMetadata;
import io.fabric8.kubernetes.api.model.VolumeBuilder;
@ -32,6 +33,7 @@ import io.quarkus.logging.Log;
import org.keycloak.operator.Config;
import org.keycloak.operator.Constants;
import org.keycloak.operator.OperatorManagedResource;
import org.keycloak.operator.StatusUpdater;
import org.keycloak.operator.v2alpha1.crds.Keycloak;
import org.keycloak.operator.v2alpha1.crds.KeycloakStatusBuilder;
@ -44,7 +46,7 @@ import java.util.Optional;
import java.util.function.Consumer;
import java.util.stream.Collectors;
public class KeycloakDeployment extends OperatorManagedResource {
public class KeycloakDeployment extends OperatorManagedResource implements StatusUpdater<KeycloakStatusBuilder> {
// public static final Pattern CONFIG_SECRET_PATTERN = Pattern.compile("^\\$\\{secret:([^:]+):(.+)}$");
@ -365,17 +367,9 @@ public class KeycloakDeployment extends OperatorManagedResource {
Container container = baseDeployment.getSpec().getTemplate().getSpec().getContainers().get(0);
container.setImage(Optional.ofNullable(keycloakCR.getSpec().getImage()).orElse(config.keycloak().image()));
var serverConfig = new HashMap<>(Constants.DEFAULT_DIST_CONFIG);
if (keycloakCR.getSpec().getServerConfiguration() != null) {
serverConfig.putAll(keycloakCR.getSpec().getServerConfiguration());
}
container.setImagePullPolicy(config.keycloak().imagePullPolicy());
container.setEnv(serverConfig.entrySet().stream()
.map(e -> new EnvVarBuilder().withName(e.getKey()).withValue(e.getValue()).build())
.collect(Collectors.toList()));
container.setEnv(getEnvVars());
addInitContainer(baseDeployment, keycloakCR.getSpec().getExtensions());
mergePodTemplate(baseDeployment.getSpec().getTemplate());
@ -406,6 +400,20 @@ public class KeycloakDeployment extends OperatorManagedResource {
return baseDeployment;
}
private List<EnvVar> getEnvVars() {
var serverConfig = new HashMap<>(Constants.DEFAULT_DIST_CONFIG);
serverConfig.put("jgroups.dns.query", getName() + Constants.KEYCLOAK_DISCOVERY_SERVICE_SUFFIX +"." + getNamespace());
if (keycloakCR.getSpec().getServerConfiguration() != null) {
serverConfig.putAll(keycloakCR.getSpec().getServerConfiguration());
}
return serverConfig.entrySet().stream()
.map(e -> new EnvVarBuilder()
.withName(e.getKey())
.withValue(e.getValue())
.build())
.collect(Collectors.toList());
}
public void updateStatus(KeycloakStatusBuilder status) {
validatePodTemplate(status);
if (existingDeployment == null) {
@ -432,14 +440,11 @@ public class KeycloakDeployment extends OperatorManagedResource {
// return configSecretsNames;
// }
@Override
public String getName() {
return keycloakCR.getMetadata().getName();
}
public String getNamespace() {
return keycloakCR.getMetadata().getNamespace();
}
public void rollingRestart() {
client.apps().deployments()
.inNamespace(getNamespace())

95
operator/src/main/java/org/keycloak/operator/v2alpha1/KeycloakDiscoveryService.java Normal file
View file

@ -0,0 +1,95 @@
/*
* Copyright 2021 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.operator.v2alpha1;
import io.fabric8.kubernetes.api.model.HasMetadata;
import io.fabric8.kubernetes.api.model.IntOrString;
import io.fabric8.kubernetes.api.model.Service;
import io.fabric8.kubernetes.api.model.ServiceBuilder;
import io.fabric8.kubernetes.api.model.ServiceSpec;
import io.fabric8.kubernetes.api.model.ServiceSpecBuilder;
import io.fabric8.kubernetes.client.KubernetesClient;
import org.keycloak.operator.Constants;
import org.keycloak.operator.OperatorManagedResource;
import org.keycloak.operator.StatusUpdater;
import org.keycloak.operator.v2alpha1.crds.Keycloak;
import org.keycloak.operator.v2alpha1.crds.KeycloakStatusBuilder;
import java.util.Optional;
public class KeycloakDiscoveryService extends OperatorManagedResource implements StatusUpdater<KeycloakStatusBuilder> {
private Service existingService;
public KeycloakDiscoveryService(KubernetesClient client, Keycloak keycloakCR) {
super(client, keycloakCR);
this.existingService = fetchExistingService();
}
private ServiceSpec getServiceSpec() {
return new ServiceSpecBuilder()
.addNewPort()
.withPort(Constants.KEYCLOAK_DISCOVERY_SERVICE_PORT)
.endPort()
.withSelector(Constants.DEFAULT_LABELS)
.withClusterIP("None")
.build();
}
@Override
protected Optional<HasMetadata> getReconciledResource() {
var service = fetchExistingService();
if (service == null) {
service = newService();
} else {
service.setSpec(getServiceSpec());
}
return Optional.of(service);
}
private Service newService() {
Service service = new ServiceBuilder()
.withNewMetadata()
.withName(getName())
.withNamespace(getNamespace())
.endMetadata()
.withSpec(getServiceSpec())
.build();
return service;
}
private Service fetchExistingService() {
return client
.services()
.inNamespace(getNamespace())
.withName(getName())
.get();
}
public void updateStatus(KeycloakStatusBuilder status) {
if (existingService == null) {
status.addNotReadyMessage("No existing Discovery Service found, waiting for creating a new one");
return;
}
}
@Override
public String getName() {
return cr.getMetadata().getName() + Constants.KEYCLOAK_DISCOVERY_SERVICE_SUFFIX;
}
}

7
operator/src/main/java/org/keycloak/operator/v2alpha1/KeycloakRealmImportJob.java
View file

@ -192,14 +192,11 @@ public class KeycloakRealmImportJob extends OperatorManagedResource {
}
}
private String getName() {
@Override
protected String getName() {
return realmCR.getMetadata().getName();
}
private String getNamespace() {
return realmCR.getMetadata().getNamespace();
}
private String getKeycloakName() { return realmCR.getSpec().getKeycloakCRName(); }
private String getRealmName() { return realmCR.getSpec().getRealm().getRealm(); }

7
operator/src/main/java/org/keycloak/operator/v2alpha1/KeycloakRealmImportSecret.java
View file

@ -48,14 +48,11 @@ public class KeycloakRealmImportSecret extends OperatorManagedResource {
.build();
}
private String getName() {
@Override
protected String getName() {
return realmCR.getMetadata().getName();
}
private String getNamespace() {
return realmCR.getMetadata().getNamespace();
}
private String getRealmName() { return realmCR.getSpec().getRealm().getRealm(); }
public String getSecretName() {

94
operator/src/main/java/org/keycloak/operator/v2alpha1/KeycloakService.java Normal file
View file

@ -0,0 +1,94 @@
/*
* Copyright 2021 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.operator.v2alpha1;
import io.fabric8.kubernetes.api.model.HasMetadata;
import io.fabric8.kubernetes.api.model.IntOrString;
import io.fabric8.kubernetes.api.model.Service;
import io.fabric8.kubernetes.api.model.ServiceBuilder;
import io.fabric8.kubernetes.api.model.ServiceSpec;
import io.fabric8.kubernetes.api.model.ServiceSpecBuilder;
import io.fabric8.kubernetes.client.KubernetesClient;
import org.keycloak.operator.Constants;
import org.keycloak.operator.OperatorManagedResource;
import org.keycloak.operator.StatusUpdater;
import org.keycloak.operator.v2alpha1.crds.Keycloak;
import org.keycloak.operator.v2alpha1.crds.KeycloakStatusBuilder;
import java.util.Optional;
public class KeycloakService extends OperatorManagedResource implements StatusUpdater<KeycloakStatusBuilder> {
private Service existingService;
public KeycloakService(KubernetesClient client, Keycloak keycloakCR) {
super(client, keycloakCR);
this.existingService = fetchExistingService();
}
private ServiceSpec getServiceSpec() {
return new ServiceSpecBuilder()
.addNewPort()
.withPort(Constants.KEYCLOAK_SERVICE_PORT)
.withProtocol(Constants.KEYCLOAK_SERVICE_PROTOCOL)
.endPort()
.withSelector(Constants.DEFAULT_LABELS)
.build();
}
@Override
protected Optional<HasMetadata> getReconciledResource() {
var service = fetchExistingService();
if (service == null) {
service = newService();
} else {
service.setSpec(getServiceSpec());
}
return Optional.of(service);
}
private Service newService() {
Service service = new ServiceBuilder()
.withNewMetadata()
.withName(getName())
.withNamespace(getNamespace())
.endMetadata()
.withSpec(getServiceSpec())
.build();
return service;
}
private Service fetchExistingService() {
return client
.services()
.inNamespace(getNamespace())
.withName(getName())
.get();
}
public void updateStatus(KeycloakStatusBuilder status) {
if (existingService == null) {
status.addNotReadyMessage("No existing Keycloak Service found, waiting for creating a new one");
return;
}
}
public String getName() {
return cr.getMetadata().getName() + Constants.KEYCLOAK_SERVICE_SUFFIX;
}
}

1
operator/src/main/kubernetes/kubernetes.yml
View file

@ -19,6 +19,7 @@ rules:
- ""
resources:
- secrets
- services
verbs:
- get
- list

6
operator/src/test/java/org/keycloak/operator/ClusterOperatorTest.java
View file

@ -108,7 +108,7 @@ public abstract class ClusterOperatorTest {
k8sclient.load(new FileInputStream(TARGET_KUBERNETES_GENERATED_YML_FOLDER +deploymentTarget+".yml"))
.inNamespace(namespace).delete();
}
private static void createCRDs() throws FileNotFoundException {
private static void createCRDs() {
Log.info("Creating CRDs");
try {
var deploymentCRD = k8sclient.load(new FileInputStream(TARGET_KUBERNETES_GENERATED_YML_FOLDER + "keycloaks.keycloak.org-v1.yml"));
@ -152,7 +152,7 @@ public abstract class ClusterOperatorTest {
protected static void deployDB() {
// DB
Log.info("Creating new PostgreSQL deployment");
k8sclient.load(KeycloakDeploymentE2EIT.class.getResourceAsStream("/example-postgres.yaml")).inNamespace(namespace).createOrReplace();
k8sclient.load(ClusterOperatorTest.class.getResourceAsStream("/example-postgres.yaml")).inNamespace(namespace).createOrReplace();
// Check DB has deployed and ready
Log.info("Checking Postgres is running");
@ -181,7 +181,7 @@ public abstract class ClusterOperatorTest {
private static void setDefaultAwaitilityTimings() {
Awaitility.setDefaultPollInterval(Duration.ofSeconds(1));
Awaitility.setDefaultTimeout(Duration.ofSeconds(180));
Awaitility.setDefaultTimeout(Duration.ofSeconds(240));
}
@AfterEach

200
operator/src/test/java/org/keycloak/operator/ClusteringE2EIT.java Normal file
View file

@ -0,0 +1,200 @@
package org.keycloak.operator;
import com.fasterxml.jackson.databind.JsonNode;
import io.fabric8.kubernetes.client.utils.Serialization;
import io.quarkus.logging.Log;
import io.quarkus.test.junit.QuarkusTest;
import io.restassured.RestAssured;
import org.awaitility.Awaitility;
import org.junit.jupiter.api.Test;
import org.keycloak.operator.utils.CRAssert;
import org.keycloak.operator.v2alpha1.KeycloakService;
import org.keycloak.operator.v2alpha1.crds.Keycloak;
import org.keycloak.operator.utils.K8sUtils;
import org.keycloak.operator.v2alpha1.crds.KeycloakRealmImport;
import org.keycloak.operator.v2alpha1.crds.KeycloakRealmImportStatusCondition;
import org.keycloak.operator.v2alpha1.crds.KeycloakStatusCondition;
import java.io.IOException;
import java.time.Duration;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.assertj.core.api.Assertions.assertThat;
@QuarkusTest
public class ClusteringE2EIT extends ClusterOperatorTest {
@Test
public void testKeycloakScaleAsExpected() {
// given
var kc = K8sUtils.getDefaultKeycloakDeployment();
var crSelector = k8sclient
.resources(Keycloak.class)
.inNamespace(kc.getMetadata().getNamespace())
.withName(kc.getMetadata().getName());
K8sUtils.deployKeycloak(k8sclient, kc, true);
var kcPodsSelector = k8sclient.pods().inNamespace(namespace).withLabel("app", "keycloak");
Keycloak keycloak = crSelector.get();
// when scale it to 10
keycloak.getSpec().setInstances(10);
k8sclient.resources(Keycloak.class).inNamespace(namespace).createOrReplace(keycloak);
Awaitility.await()
.atMost(1, MINUTES)
.pollDelay(1, SECONDS)
.ignoreExceptions()
.untilAsserted(() -> CRAssert.assertKeycloakStatusCondition(crSelector.get(), KeycloakStatusCondition.READY, false));
Awaitility.await()
.atMost(Duration.ofSeconds(5))
.untilAsserted(() -> assertThat(kcPodsSelector.list().getItems().size()).isEqualTo(10));
// when scale it down to 2
keycloak.getSpec().setInstances(2);
k8sclient.resources(Keycloak.class).inNamespace(namespace).createOrReplace(keycloak);
Awaitility.await()
.atMost(Duration.ofSeconds(180))
.untilAsserted(() -> assertThat(kcPodsSelector.list().getItems().size()).isEqualTo(2));
Awaitility.await()
.atMost(2, MINUTES)
.pollDelay(5, SECONDS)
.ignoreExceptions()
.untilAsserted(() -> CRAssert.assertKeycloakStatusCondition(crSelector.get(), KeycloakStatusCondition.READY, true));
// get the service
var service = new KeycloakService(k8sclient, kc);
String url = "http://" + service.getName() + "." + namespace + ":" + Constants.KEYCLOAK_SERVICE_PORT;
Awaitility.await().atMost(5, MINUTES).untilAsserted(() -> {
Log.info("Starting curl Pod to test if the realm is available");
Log.info("Url: '" + url + "'");
String curlOutput = K8sUtils.inClusterCurl(k8sclient, namespace, url);
Log.info("Output from curl: '" + curlOutput + "'");
assertThat(curlOutput).isEqualTo("200");
});
}
// local debug commands:
// export TOKEN=$(curl --data "grant_type=password&client_id=token-test-client&username=test&password=test" http://localhost:8080/realms/token-test/protocol/openid-connect/token | jq -r '.access_token')
//
// curl http://localhost:8080/realms/token-test/protocol/openid-connect/userinfo -H "Authorization: bearer $TOKEN"
//
// example good answer:
// {"sub":"b660eec6-a93b-46fd-abb2-e9fbdff67a63","email_verified":false,"preferred_username":"test"}
// example error answer:
// {"error":"invalid_request","error_description":"Token not provided"}
@Test
public void testKeycloakCacheIsConnected() throws Exception {
// given
Log.info("Setup");
var kc = K8sUtils.getDefaultKeycloakDeployment();
var crSelector = k8sclient
.resources(Keycloak.class)
.inNamespace(kc.getMetadata().getNamespace())
.withName(kc.getMetadata().getName());
var targetInstances = 3;
kc.getSpec().setInstances(targetInstances);
k8sclient.resources(Keycloak.class).inNamespace(namespace).createOrReplace(kc);
var realm = k8sclient.resources(KeycloakRealmImport.class).inNamespace(namespace).load(getClass().getResourceAsStream("/token-test-realm.yaml"));
var realmImportSelector = k8sclient.resources(KeycloakRealmImport.class).inNamespace(namespace).withName("example-token-test-kc");
realm.createOrReplace();
Log.info("Waiting for a stable Keycloak Cluster");
Awaitility.await()
.atMost(10, MINUTES)
.pollDelay(5, SECONDS)
.ignoreExceptions()
.untilAsserted(() -> {
Log.info("Checking realm import has finished.");
CRAssert.assertKeycloakRealmImportStatusCondition(realmImportSelector.get(), KeycloakRealmImportStatusCondition.DONE, true);
Log.info("Checking Keycloak is stable.");
CRAssert.assertKeycloakStatusCondition(crSelector.get(), KeycloakStatusCondition.READY, true);
});
Log.info("Testing the Keycloak Cluster");
Awaitility.await().atMost(5, MINUTES).ignoreExceptions().untilAsserted(() -> {
// Get the list of Keycloak pods
var pods = k8sclient
.pods()
.inNamespace(namespace)
.withLabels(Constants.DEFAULT_LABELS)
.list()
.getItems();
String token = null;
// Obtaining the token from the first pod
// Connecting using port-forward and a fixed port to respect the instance issuer used hostname
for (var pod: pods) {
Log.info("Testing Pod: " + pod.getMetadata().getName());
try (var portForward = k8sclient
.pods()
.inNamespace(namespace)
.withName(pod.getMetadata().getName())
.portForward(8080, 8080)) {
token = (token != null) ? token : RestAssured.given()
.param("grant_type" , "password")
.param("client_id", "token-test-client")
.param("username", "test")
.param("password", "test")
.post("http://localhost:" + portForward.getLocalPort() + "/realms/token-test/protocol/openid-connect/token")
.body()
.jsonPath()
.getString("access_token");
Log.info("Using token:" + token);
var username = RestAssured.given()
.header("Authorization", "Bearer " + token)
.get("http://localhost:" + portForward.getLocalPort() + "/realms/token-test/protocol/openid-connect/userinfo")
.body()
.jsonPath()
.getString("preferred_username");
Log.info("Username found: " + username);
assertThat(username).isEqualTo("test");
}
}
});
// This is to test passing through the "Service", not 100% deterministic, but a smoke test that things are working as expected
// Executed here to avoid paying the setup time again
var service = new KeycloakService(k8sclient, kc);
Awaitility.await().atMost(5, MINUTES).ignoreExceptions().untilAsserted(() -> {
String token2 = null;
// Obtaining the token from the first pod
// Connecting using port-forward and a fixed port to respect the instance issuer used hostname
for (int i = 0; i < (targetInstances * 2); i++) {
if (token2 == null) {
var tokenUrl = "http://" + service.getName() + "." + namespace + ":" + Constants.KEYCLOAK_SERVICE_PORT + "/realms/token-test/protocol/openid-connect/token";
Log.info("Checking url: " + tokenUrl);
var tokenOutput = K8sUtils.inClusterCurl(k8sclient, namespace, "-s", "--data", "grant_type=password&client_id=token-test-client&username=test&password=test", tokenUrl);
Log.info("Curl Output with token: " + tokenOutput);
JsonNode tokenAnswer = Serialization.jsonMapper().readTree(tokenOutput);
assertThat(tokenAnswer.hasNonNull("access_token")).isTrue();
token2 = tokenAnswer.get("access_token").asText();
}
String url = "http://" + service.getName() + "." + namespace + ":" + Constants.KEYCLOAK_SERVICE_PORT + "/realms/token-test/protocol/openid-connect/userinfo";
Log.info("Checking url: " + url);
var curlOutput = K8sUtils.inClusterCurl(k8sclient, namespace, "-s", "-H", "Authorization: Bearer " + token2, url);
Log.info("Curl Output on access attempt: " + curlOutput);
JsonNode answer = Serialization.jsonMapper().readTree(curlOutput);
assertThat(answer.hasNonNull("preferred_username")).isTrue();
assertThat(answer.get("preferred_username").asText()).isEqualTo("test");
}
});
}
}

95
operator/src/test/java/org/keycloak/operator/KeycloakServicesE2EIT.java Normal file
View file

@ -0,0 +1,95 @@
package org.keycloak.operator;
import io.fabric8.kubernetes.api.model.ServiceSpecBuilder;
import io.quarkus.logging.Log;
import io.quarkus.test.junit.QuarkusTest;
import org.awaitility.Awaitility;
import org.junit.jupiter.api.Test;
import org.keycloak.operator.v2alpha1.KeycloakDiscoveryService;
import org.keycloak.operator.v2alpha1.KeycloakService;
import org.keycloak.operator.utils.K8sUtils;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
@QuarkusTest
public class KeycloakServicesE2EIT extends ClusterOperatorTest {
@Test
public void testMainServiceDurability() {
var kc = K8sUtils.getDefaultKeycloakDeployment();
K8sUtils.deployKeycloak(k8sclient, kc, true);
var service = new KeycloakService(k8sclient, kc);
var serviceSelector = k8sclient.services().inNamespace(namespace).withName(service.getName());
Log.info("Trying to delete the service");
assertThat(serviceSelector.delete()).isTrue();
Awaitility.await()
.untilAsserted(() -> assertThat(serviceSelector.get()).isNotNull());
K8sUtils.waitForKeycloakToBeReady(k8sclient, kc); // wait for reconciler to calm down to avoid race condititon
Log.info("Trying to modify the service");
var currentService = serviceSelector.get();
var labels = Map.of("address", "EvergreenTerrace742");
// ignoring current IP/s
currentService.getSpec().setClusterIP(null);
currentService.getSpec().setClusterIPs(null);
var origSpecs = new ServiceSpecBuilder(currentService.getSpec()).build(); // deep copy
currentService.getMetadata().getLabels().putAll(labels);
currentService.getSpec().setSessionAffinity("ClientIP");
serviceSelector.createOrReplace(currentService);
Awaitility.await()
.untilAsserted(() -> {
var s = serviceSelector.get();
assertThat(s.getMetadata().getLabels().entrySet().containsAll(labels.entrySet())).isTrue(); // additional labels should not be overwritten
// ignoring assigned IP/s
s.getSpec().setClusterIP(null);
s.getSpec().setClusterIPs(null);
assertThat(s.getSpec()).isEqualTo(origSpecs); // specs should be reconciled back to original values
});
}
@Test
public void testDiscoveryServiceDurability() {
var kc = K8sUtils.getDefaultKeycloakDeployment();
K8sUtils.deployKeycloak(k8sclient, kc, true);
var discoveryService = new KeycloakDiscoveryService(k8sclient, kc);
var discoveryServiceSelector = k8sclient.services().inNamespace(namespace).withName(discoveryService.getName());
Log.info("Trying to delete the discovery service");
assertThat(discoveryServiceSelector.delete()).isTrue();
Awaitility.await()
.untilAsserted(() -> assertThat(discoveryServiceSelector.get()).isNotNull());
K8sUtils.waitForKeycloakToBeReady(k8sclient, kc); // wait for reconciler to calm down to avoid race condititon
Log.info("Trying to modify the service");
var currentDiscoveryService = discoveryServiceSelector.get();
var labels = Map.of("address", "EvergreenTerrace742");
// ignoring current IP/s
currentDiscoveryService.getSpec().setClusterIP(null);
currentDiscoveryService.getSpec().setClusterIPs(null);
var origDiscoverySpecs = new ServiceSpecBuilder(currentDiscoveryService.getSpec()).build(); // deep copy
currentDiscoveryService.getMetadata().getLabels().putAll(labels);
currentDiscoveryService.getSpec().setSessionAffinity("ClientIP");
discoveryServiceSelector.createOrReplace(currentDiscoveryService);
Awaitility.await()
.untilAsserted(() -> {
var ds = discoveryServiceSelector.get();
assertThat(ds.getMetadata().getLabels().entrySet().containsAll(labels.entrySet())).isTrue(); // additional labels should not be overwritten
// ignoring assigned IP/s
ds.getSpec().setClusterIP(null);
ds.getSpec().setClusterIPs(null);
assertThat(ds.getSpec()).isEqualTo(origDiscoverySpecs); // specs should be reconciled back to original values
});
}
}

121
operator/src/test/java/org/keycloak/operator/RealmImportE2EIT.java
View file

@ -1,22 +1,19 @@
package org.keycloak.operator;
import io.fabric8.kubernetes.api.model.Pod;
import io.fabric8.kubernetes.api.model.ServiceBuilder;
import io.fabric8.kubernetes.client.KubernetesClientException;
import io.fabric8.kubernetes.client.extended.run.RunConfigBuilder;
import io.quarkus.logging.Log;
import io.quarkus.test.junit.QuarkusTest;
import org.awaitility.Awaitility;
import org.junit.jupiter.api.Test;
import org.keycloak.operator.utils.CRAssert;
import org.keycloak.operator.v2alpha1.KeycloakService;
import org.keycloak.operator.v2alpha1.crds.KeycloakRealmImport;
import org.keycloak.operator.v2alpha1.crds.KeycloakRealmImportStatusCondition;
import java.util.List;
import java.util.Map;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.assertj.core.api.Assertions.assertThat;
import static org.keycloak.operator.Constants.KEYCLOAK_SERVICE_PORT;
import static org.keycloak.operator.utils.K8sUtils.getDefaultKeycloakDeployment;
import static org.keycloak.operator.utils.K8sUtils.inClusterCurl;
import static org.keycloak.operator.v2alpha1.crds.KeycloakRealmImportStatusCondition.DONE;
import static org.keycloak.operator.v2alpha1.crds.KeycloakRealmImportStatusCondition.STARTED;
import static org.keycloak.operator.v2alpha1.crds.KeycloakRealmImportStatusCondition.HAS_ERRORS;
@ -24,39 +21,11 @@ import static org.keycloak.operator.v2alpha1.crds.KeycloakRealmImportStatusCondi
@QuarkusTest
public class RealmImportE2EIT extends ClusterOperatorTest {
final static String KEYCLOAK_SERVICE_NAME = "example-keycloak";
final static int KEYCLOAK_PORT = 8080;
private KeycloakRealmImportStatusCondition getCondition(List<KeycloakRealmImportStatusCondition> conditions, String type) {
return conditions
.stream()
.filter(c -> c.getType().equals(type))
.findFirst()
.get();
}
@Test
public void testWorkingRealmImport() {
Log.info(((operatorDeployment == OperatorDeployment.remote) ? "Remote " : "Local ") + "Run Test :" + namespace);
// Arrange
k8sclient.load(getClass().getResourceAsStream("/example-postgres.yaml")).inNamespace(namespace).createOrReplace();
k8sclient.load(getClass().getResourceAsStream("/example-keycloak.yml")).inNamespace(namespace).createOrReplace();
k8sclient.services().inNamespace(namespace).create(
new ServiceBuilder()
.withNewMetadata()
.withName(KEYCLOAK_SERVICE_NAME)
.withNamespace(namespace)
.endMetadata()
.withNewSpec()
.withSelector(Map.of("app", "keycloak"))
.addNewPort()
.withPort(KEYCLOAK_PORT)
.endPort()
.endSpec()
.build()
);
// Act
k8sclient.load(getClass().getResourceAsStream("/example-realm.yaml")).inNamespace(namespace).createOrReplace();
@ -70,14 +39,9 @@ public class RealmImportE2EIT extends ClusterOperatorTest {
.pollDelay(5, SECONDS)
.ignoreExceptions()
.untilAsserted(() -> {
var conditions = crSelector
.get()
.getStatus()
.getConditions();
assertThat(getCondition(conditions, DONE).getStatus()).isFalse();
assertThat(getCondition(conditions, STARTED).getStatus()).isTrue();
assertThat(getCondition(conditions, HAS_ERRORS).getStatus()).isFalse();
CRAssert.assertKeycloakRealmImportStatusCondition(crSelector.get(), DONE, false);
CRAssert.assertKeycloakRealmImportStatusCondition(crSelector.get(), STARTED, true);
CRAssert.assertKeycloakRealmImportStatusCondition(crSelector.get(), HAS_ERRORS, false);
});
Awaitility.await()
@ -85,62 +49,26 @@ public class RealmImportE2EIT extends ClusterOperatorTest {
.pollDelay(5, SECONDS)
.ignoreExceptions()
.untilAsserted(() -> {
var conditions = crSelector
.get()
.getStatus()
.getConditions();
assertThat(getCondition(conditions, DONE).getStatus()).isTrue();
assertThat(getCondition(conditions, STARTED).getStatus()).isFalse();
assertThat(getCondition(conditions, HAS_ERRORS).getStatus()).isFalse();
CRAssert.assertKeycloakRealmImportStatusCondition(crSelector.get(), DONE, true);
CRAssert.assertKeycloakRealmImportStatusCondition(crSelector.get(), STARTED, false);
CRAssert.assertKeycloakRealmImportStatusCondition(crSelector.get(), HAS_ERRORS, false);
});
var service = new KeycloakService(k8sclient, getDefaultKeycloakDeployment());
String url =
"http://" + KEYCLOAK_SERVICE_NAME + "." + namespace + ":" + KEYCLOAK_PORT + "/realms/count0";
"http://" + service.getName() + "." + namespace + ":" + KEYCLOAK_SERVICE_PORT + "/realms/count0";
Awaitility.await().atMost(5, MINUTES).untilAsserted(() -> {
try {
Log.info("Starting curl Pod to test if the realm is available");
Pod curlPod = k8sclient.run().inNamespace(namespace)
.withRunConfig(new RunConfigBuilder()
.withArgs("-s", "-o", "/dev/null", "-w", "%{http_code}", url)
.withName("curl")
.withImage("curlimages/curl:7.78.0")
.withRestartPolicy("Never")
.build())
.done();
Log.info("Waiting for curl Pod to finish running");
Awaitility.await().atMost(2, MINUTES)
.until(() -> {
String phase =
k8sclient.pods().inNamespace(namespace).withName("curl").get()
.getStatus().getPhase();
return phase.equals("Succeeded") || phase.equals("Failed");
});
String curlOutput =
k8sclient.pods().inNamespace(namespace)
.withName(curlPod.getMetadata().getName()).getLog();
Log.info("Output from curl: '" + curlOutput + "'");
assertThat(curlOutput).isEqualTo("200");
} catch (KubernetesClientException ex) {
throw new AssertionError(ex);
} finally {
Log.info("Deleting curl Pod");
k8sclient.pods().inNamespace(namespace).withName("curl").delete();
Awaitility.await().atMost(1, MINUTES)
.until(() -> k8sclient.pods().inNamespace(namespace).withName("curl")
.get() == null);
}
Log.info("Starting curl Pod to test if the realm is available");
Log.info("Url: '" + url + "'");
String curlOutput = inClusterCurl(k8sclient, namespace, url);
Log.info("Output from curl: '" + curlOutput + "'");
assertThat(curlOutput).isEqualTo("200");
});
}
@Test
public void testNotWorkingRealmImport() {
Log.info(((operatorDeployment == OperatorDeployment.remote) ? "Remote " : "Local ") + "Run Test :" + namespace);
// Arrange
k8sclient.load(getClass().getResourceAsStream("/example-postgres.yaml")).inNamespace(namespace).createOrReplace();
k8sclient.load(getClass().getResourceAsStream("/example-keycloak.yml")).inNamespace(namespace).createOrReplace();
// Act
@ -152,17 +80,14 @@ public class RealmImportE2EIT extends ClusterOperatorTest {
.pollDelay(5, SECONDS)
.ignoreExceptions()
.untilAsserted(() -> {
var conditions = k8sclient
var crSelector = k8sclient
.resources(KeycloakRealmImport.class)
.inNamespace(namespace)
.withName("example-count0-kc")
.get()
.getStatus()
.getConditions();
.withName("example-count0-kc");
assertThat(getCondition(conditions, HAS_ERRORS).getStatus()).isTrue();
assertThat(getCondition(conditions, DONE).getStatus()).isFalse();
assertThat(getCondition(conditions, STARTED).getStatus()).isFalse();
CRAssert.assertKeycloakRealmImportStatusCondition(crSelector.get(), DONE, false);
CRAssert.assertKeycloakRealmImportStatusCondition(crSelector.get(), STARTED, false);
CRAssert.assertKeycloakRealmImportStatusCondition(crSelector.get(), HAS_ERRORS, true);
});
}

5
operator/src/test/java/org/keycloak/operator/utils/CRAssert.java
View file

@ -38,4 +38,9 @@ public final class CRAssert {
(containedMessage == null || c.getMessage().contains(containedMessage)))
).isTrue();
}
public static void assertKeycloakRealmImportStatusCondition(KeycloakRealmImport kri, String condition, boolean status) {
assertThat(kri.getStatus().getConditions().stream()
.anyMatch(c -> c.getType().equals(condition) && c.getStatus() == status)).isTrue();
}
}

49
operator/src/test/java/org/keycloak/operator/utils/K8sUtils.java
View file

@ -17,8 +17,13 @@
package org.keycloak.operator.utils;
import io.fabric8.kubernetes.api.model.Pod;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.KubernetesClientException;
import io.fabric8.kubernetes.client.extended.run.RunConfigBuilder;
import io.fabric8.kubernetes.client.utils.KubernetesResourceUtil;
import io.fabric8.kubernetes.client.utils.Serialization;
import io.quarkus.kubernetes.client.runtime.KubernetesClientUtils;
import io.quarkus.logging.Log;
import org.awaitility.Awaitility;
import org.keycloak.operator.v2alpha1.crds.Keycloak;
@ -27,6 +32,10 @@ import org.keycloak.operator.v2alpha1.crds.KeycloakStatusCondition;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.UUID;
import static java.util.concurrent.TimeUnit.MINUTES;
import static org.assertj.core.api.Assertions.assertThat;
/**
* @author Vaclav Muzikar <vmuzikar@redhat.com>
@ -67,4 +76,44 @@ public final class K8sUtils {
CRAssert.assertKeycloakStatusCondition(currentKc, KeycloakStatusCondition.HAS_ERRORS, false);
});
}
public static String inClusterCurl(KubernetesClient k8sclient, String namespace, String url) {
return inClusterCurl(k8sclient, namespace, "-s", "-o", "/dev/null", "-w", "%{http_code}", url);
}
public static String inClusterCurl(KubernetesClient k8sclient, String namespace, String... args) {
var podName = KubernetesResourceUtil.sanitizeName("curl-" + UUID.randomUUID());
try {
Pod curlPod = k8sclient.run().inNamespace(namespace)
.withRunConfig(new RunConfigBuilder()
.withArgs(args)
.withName(podName)
.withImage("curlimages/curl:7.78.0")
.withRestartPolicy("Never")
.build())
.done();
Log.info("Waiting for curl Pod to finish running");
Awaitility.await().atMost(2, MINUTES)
.until(() -> {
String phase =
k8sclient.pods().inNamespace(namespace).withName(podName).get()
.getStatus().getPhase();
return phase.equals("Succeeded") || phase.equals("Failed");
});
String curlOutput =
k8sclient.pods().inNamespace(namespace)
.withName(curlPod.getMetadata().getName()).getLog();
return curlOutput;
} catch (KubernetesClientException ex) {
throw new AssertionError(ex);
} finally {
Log.info("Deleting curl Pod");
k8sclient.pods().inNamespace(namespace).withName(podName).delete();
Awaitility.await().atMost(1, MINUTES)
.until(() -> k8sclient.pods().inNamespace(namespace).withName(podName)
.get() == null);
}
}
}

1725
operator/src/test/resources/token-test-realm.yaml Normal file
View file

File diff suppressed because it is too large Load diff