Switch to StatefulSet (#12757)

This commit is contained in:
Andrea Peruffo 2022-07-13 15:58:06 +01:00 committed by GitHub
parent 46b4b0851d
commit f2d71cd1c7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 158 additions and 99 deletions

View file

@ -17,7 +17,7 @@
package org.keycloak.operator.controllers;
import io.fabric8.kubernetes.api.model.Service;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.api.model.apps.StatefulSet;
import io.fabric8.kubernetes.api.model.networking.v1.Ingress;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.informers.SharedIndexInformer;
@ -61,8 +61,8 @@ public class KeycloakController implements Reconciler<Keycloak>, EventSourceInit
public List<EventSource> prepareEventSources(EventSourceContext<Keycloak> context) {
String namespace = context.getConfigurationService().getClientConfiguration().getNamespace();
SharedIndexInformer<Deployment> deploymentInformer =
client.apps().deployments().inNamespace(namespace)
SharedIndexInformer<StatefulSet> deploymentInformer =
client.apps().statefulSets().inNamespace(namespace)
.withLabels(Constants.DEFAULT_LABELS)
.runnableInformer(0);

View file

@ -22,13 +22,12 @@ import io.fabric8.kubernetes.api.model.EnvVarBuilder;
import io.fabric8.kubernetes.api.model.EnvVarSourceBuilder;
import io.fabric8.kubernetes.api.model.ExecActionBuilder;
import io.fabric8.kubernetes.api.model.HasMetadata;
import io.fabric8.kubernetes.api.model.IntOrString;
import io.fabric8.kubernetes.api.model.PodTemplateSpec;
import io.fabric8.kubernetes.api.model.ResourceRequirements;
import io.fabric8.kubernetes.api.model.VolumeBuilder;
import io.fabric8.kubernetes.api.model.VolumeMountBuilder;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.api.model.apps.DeploymentBuilder;
import io.fabric8.kubernetes.api.model.apps.StatefulSet;
import io.fabric8.kubernetes.api.model.apps.StatefulSetBuilder;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.quarkus.logging.Log;
import org.keycloak.operator.Config;
@ -55,13 +54,13 @@ public class KeycloakDeployment extends OperatorManagedResource implements Statu
private final Config config;
private final Keycloak keycloakCR;
private final Deployment existingDeployment;
private final Deployment baseDeployment;
private final StatefulSet existingDeployment;
private final StatefulSet baseDeployment;
private final String adminSecretName;
private Set<String> serverConfigSecretsNames;
public KeycloakDeployment(KubernetesClient client, Config config, Keycloak keycloakCR, Deployment existingDeployment, String adminSecretName) {
public KeycloakDeployment(KubernetesClient client, Config config, Keycloak keycloakCR, StatefulSet existingDeployment, String adminSecretName) {
super(client, keycloakCR);
this.config = config;
this.keycloakCR = keycloakCR;
@ -81,15 +80,15 @@ public class KeycloakDeployment extends OperatorManagedResource implements Statu
@Override
public Optional<HasMetadata> getReconciledResource() {
Deployment baseDeployment = new DeploymentBuilder(this.baseDeployment).build(); // clone not to change the base template
Deployment reconciledDeployment;
StatefulSet baseDeployment = new StatefulSetBuilder(this.baseDeployment).build(); // clone not to change the base template
StatefulSet reconciledDeployment;
if (existingDeployment == null) {
Log.info("No existing Deployment found, using the default");
reconciledDeployment = baseDeployment;
}
else {
Log.info("Existing Deployment found, updating specs");
reconciledDeployment = new DeploymentBuilder(existingDeployment).build();
reconciledDeployment = new StatefulSetBuilder(existingDeployment).build();
// don't overwrite metadata, just specs
reconciledDeployment.setSpec(baseDeployment.getSpec());
@ -106,10 +105,10 @@ public class KeycloakDeployment extends OperatorManagedResource implements Statu
return Optional.of(reconciledDeployment);
}
private Deployment fetchExistingDeployment() {
private StatefulSet fetchExistingDeployment() {
return client
.apps()
.deployments()
.statefulSets()
.inNamespace(getNamespace())
.withName(getName())
.get();
@ -319,7 +318,7 @@ public class KeycloakDeployment extends OperatorManagedResource implements Statu
}
}
private void configureHostname(Deployment deployment) {
private void configureHostname(StatefulSet deployment) {
var kcContainer = deployment.getSpec().getTemplate().getSpec().getContainers().get(0);
var hostname = this.keycloakCR.getSpec().getHostname();
var envVars = kcContainer.getEnv();
@ -346,7 +345,7 @@ public class KeycloakDeployment extends OperatorManagedResource implements Statu
}
}
private void configureTLS(Deployment deployment) {
private void configureTLS(StatefulSet deployment) {
var kcContainer = deployment.getSpec().getTemplate().getSpec().getContainers().get(0);
var tlsSecret = this.keycloakCR.getSpec().getTlsSecret();
var envVars = kcContainer.getEnv();
@ -462,8 +461,8 @@ public class KeycloakDeployment extends OperatorManagedResource implements Statu
}
}
private Deployment createBaseDeployment() {
Deployment baseDeployment = new DeploymentBuilder()
private StatefulSet createBaseDeployment() {
StatefulSet baseDeployment = new StatefulSetBuilder()
.withNewMetadata()
.endMetadata()
.withNewSpec()
@ -502,13 +501,6 @@ public class KeycloakDeployment extends OperatorManagedResource implements Statu
.endContainer()
.endSpec()
.endTemplate()
.withNewStrategy()
.withNewRollingUpdate()
// Same defaults as for a StatefulSet: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#maximum-unavailable-pods
.withNewMaxSurge(1) // maximum number of Pods that can be created over the desired number of Pods
.withNewMaxUnavailable(1) // maximum number of Pods that can be unavailable during the update process
.endRollingUpdate()
.endStrategy()
.endSpec()
.build();
@ -601,15 +593,7 @@ public class KeycloakDeployment extends OperatorManagedResource implements Statu
public void updateStatus(KeycloakStatusBuilder status) {
validatePodTemplate(status);
if (existingDeployment == null) {
status.addNotReadyMessage("No existing Deployment found, waiting for creating a new one");
return;
}
var replicaFailure = existingDeployment.getStatus().getConditions().stream()
.filter(d -> d.getType().equals("ReplicaFailure")).findFirst();
if (replicaFailure.isPresent()) {
status.addNotReadyMessage("Deployment failures");
status.addErrorMessage("Deployment failure: " + replicaFailure.get());
status.addNotReadyMessage("No existing StatefulSet found, waiting for creating a new one");
return;
}
@ -619,16 +603,12 @@ public class KeycloakDeployment extends OperatorManagedResource implements Statu
status.addNotReadyMessage("Waiting for more replicas");
}
var progressing = existingDeployment.getStatus().getConditions().stream()
.filter(c -> c.getType().equals("Progressing")).findFirst();
progressing.ifPresent(p -> {
String reason = p.getReason();
// https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#progressing-deployment
if (p.getStatus().equals("True") &&
(reason.equals("NewReplicaSetCreated") || reason.equals("FoundNewReplicaSet") || reason.equals("ReplicaSetUpdated"))) {
if (existingDeployment.getStatus() != null
&& existingDeployment.getStatus().getCurrentRevision() != null
&& existingDeployment.getStatus().getUpdateRevision() != null
&& !existingDeployment.getStatus().getCurrentRevision().equals(existingDeployment.getStatus().getUpdateRevision())) {
status.addRollingUpdateMessage("Rolling out deployment update");
}
});
}
public Set<String> getConfigSecretsNames() {
@ -645,7 +625,7 @@ public class KeycloakDeployment extends OperatorManagedResource implements Statu
}
public void rollingRestart() {
client.apps().deployments()
client.apps().statefulSets()
.inNamespace(getNamespace())
.withName(getName())
.rolling().restart();

View file

@ -17,13 +17,14 @@
package org.keycloak.operator.controllers;
import io.fabric8.kubernetes.api.model.Container;
import io.fabric8.kubernetes.api.model.EnvVarBuilder;
import io.fabric8.kubernetes.api.model.HasMetadata;
import io.fabric8.kubernetes.api.model.PodTemplateSpec;
import io.fabric8.kubernetes.api.model.SecretVolumeSourceBuilder;
import io.fabric8.kubernetes.api.model.Volume;
import io.fabric8.kubernetes.api.model.VolumeBuilder;
import io.fabric8.kubernetes.api.model.VolumeMountBuilder;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.api.model.apps.StatefulSet;
import io.fabric8.kubernetes.api.model.batch.v1.Job;
import io.fabric8.kubernetes.api.model.batch.v1.JobBuilder;
import io.fabric8.kubernetes.client.KubernetesClient;
@ -37,11 +38,14 @@ import org.keycloak.operator.crds.v2alpha1.realmimport.KeycloakRealmImportStatus
import java.util.List;
import java.util.Optional;
import static org.keycloak.operator.Constants.DEFAULT_DIST_CONFIG;
import static org.keycloak.operator.controllers.KeycloakDeployment.getEnvVarName;
public class KeycloakRealmImportJob extends OperatorManagedResource {
private final Keycloak keycloak;
private final KeycloakRealmImport realmCR;
private final Deployment existingDeployment;
private final StatefulSet existingDeployment;
private final Job existingJob;
private final String secretName;
private final String volumeName;
@ -80,10 +84,10 @@ public class KeycloakRealmImportJob extends OperatorManagedResource {
.get();
}
private Deployment fetchExistingDeployment() {
private StatefulSet fetchExistingDeployment() {
return client
.apps()
.deployments()
.statefulSets()
.inNamespace(getNamespace())
.withName(getKeycloakName())
.get();
@ -129,6 +133,26 @@ public class KeycloakRealmImportJob extends OperatorManagedResource {
buildKeycloakJobContainer(keycloakPodTemplate.getSpec().getContainers().get(0));
keycloakPodTemplate.getSpec().getVolumes().add(buildSecretVolume());
var labels = keycloakPodTemplate.getMetadata().getLabels();
// The Job should not be selected with app=keycloak
labels.put("app", "keycloak-realm-import");
var envvars = keycloakPodTemplate
.getSpec()
.getContainers()
.get(0)
.getEnv();
var cacheEnvVarName = getEnvVarName("cache");
var healthEnvVarName = getEnvVarName("health-enabled");
envvars.removeIf(e -> e.getName().equals(cacheEnvVarName) || e.getName().equals(healthEnvVarName));
// The Job should not connect to the cache
envvars.add(new EnvVarBuilder().withName(cacheEnvVarName).withValue("local").build());
// The Job doesn't need health to be enabled
envvars.add(new EnvVarBuilder().withName(healthEnvVarName).withValue("false").build());
return buildJob(keycloakPodTemplate);
}
@ -205,7 +229,7 @@ public class KeycloakRealmImportJob extends OperatorManagedResource {
private String getRealmName() { return realmCR.getSpec().getRealm().getRealm(); }
private void rollingRestart() {
client.apps().deployments()
client.apps().statefulSets()
.inNamespace(getNamespace())
.withName(getKeycloakName())
.rolling().restart();

View file

@ -9,7 +9,7 @@ rules:
# https://github.com/fabric8io/kubernetes-client/issues/3996
- extensions
resources:
- deployments
- statefulsets
verbs:
- get
- list

View file

@ -20,7 +20,6 @@ package org.keycloak.operator.testsuite.integration;
import io.fabric8.kubernetes.api.model.HasMetadata;
import io.fabric8.kubernetes.api.model.NamespaceBuilder;
import io.fabric8.kubernetes.api.model.Secret;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.client.Config;
import io.fabric8.kubernetes.client.ConfigBuilder;
import io.fabric8.kubernetes.client.DefaultKubernetesClient;
@ -186,6 +185,17 @@ public abstract class BaseOperatorTest {
protected static void deleteDB() {
// Delete the Postgres StatefulSet
k8sclient.apps().statefulSets().inNamespace(namespace).withName("postgresql-db").delete();
Awaitility.await()
.ignoreExceptions()
.untilAsserted(() -> {
Log.infof("Waiting for postgres to be deleted");
assertThat(k8sclient
.apps()
.statefulSets()
.inNamespace(namespace)
.withName("postgresql-db")
.get()).isNull();
});
}
// TODO improve this (preferably move to JOSDK)
@ -220,7 +230,7 @@ public abstract class BaseOperatorTest {
.untilAsserted(() -> {
var kcDeployments = k8sclient
.apps()
.deployments()
.statefulSets()
.inNamespace(namespace)
.withLabels(Constants.DEFAULT_LABELS)
.list()

View file

@ -20,7 +20,7 @@ package org.keycloak.operator.testsuite.integration;
import io.fabric8.kubernetes.api.model.EnvVarBuilder;
import io.fabric8.kubernetes.api.model.SecretBuilder;
import io.fabric8.kubernetes.api.model.SecretKeySelectorBuilder;
import io.fabric8.kubernetes.api.model.apps.DeploymentSpecBuilder;
import io.fabric8.kubernetes.api.model.apps.StatefulSetSpecBuilder;
import io.quarkus.logging.Log;
import io.quarkus.test.junit.QuarkusTest;
import org.awaitility.Awaitility;
@ -65,17 +65,17 @@ public class KeycloakDeploymentTest extends BaseOperatorTest {
// Check Operator has deployed Keycloak
Log.info("Checking Operator has deployed Keycloak deployment");
assertThat(k8sclient.apps().deployments().inNamespace(namespace).withName(deploymentName).get()).isNotNull();
assertThat(k8sclient.apps().statefulSets().inNamespace(namespace).withName(deploymentName).get()).isNotNull();
// Check Keycloak has correct replicas
Log.info("Checking Keycloak pod has ready replicas == 1");
assertThat(k8sclient.apps().deployments().inNamespace(namespace).withName(deploymentName).get().getStatus().getReadyReplicas()).isEqualTo(1);
assertThat(k8sclient.apps().statefulSets().inNamespace(namespace).withName(deploymentName).get().getStatus().getReadyReplicas()).isEqualTo(1);
// Delete CR
Log.info("Deleting Keycloak CR and watching cleanup");
k8sclient.resources(Keycloak.class).delete(kc);
Awaitility.await()
.untilAsserted(() -> assertThat(k8sclient.apps().deployments().inNamespace(namespace).withName(deploymentName).get()).isNull());
.untilAsserted(() -> assertThat(k8sclient.apps().statefulSets().inNamespace(namespace).withName(deploymentName).get()).isNull());
} catch (Exception e) {
savePodLogs();
throw e;
@ -99,7 +99,7 @@ public class KeycloakDeploymentTest extends BaseOperatorTest {
Awaitility.await()
.during(Duration.ofSeconds(15)) // check if the Deployment is stable
.untilAsserted(() -> {
var c = k8sclient.apps().deployments().inNamespace(namespace).withName(deploymentName).get()
var c = k8sclient.apps().statefulSets().inNamespace(namespace).withName(deploymentName).get()
.getSpec().getTemplate().getSpec().getContainers().get(0);
assertThat(c.getImage()).isEqualTo("quay.io/keycloak/non-existing-keycloak");
assertThat(c.getEnv().stream()
@ -132,7 +132,7 @@ public class KeycloakDeploymentTest extends BaseOperatorTest {
.ignoreExceptions()
.untilAsserted(() -> {
Log.info("Asserting default value was overwritten by CR value");
var c = k8sclient.apps().deployments().inNamespace(namespace).withName(kc.getMetadata().getName()).get()
var c = k8sclient.apps().statefulSets().inNamespace(namespace).withName(kc.getMetadata().getName()).get()
.getSpec().getTemplate().getSpec().getContainers().get(0);
assertThat(c.getEnv()).contains(e);
@ -151,29 +151,29 @@ public class KeycloakDeploymentTest extends BaseOperatorTest {
deployKeycloak(k8sclient, kc, true);
Log.info("Trying to delete deployment");
assertThat(k8sclient.apps().deployments().withName(deploymentName).delete()).isTrue();
assertThat(k8sclient.apps().statefulSets().withName(deploymentName).delete()).isTrue();
Awaitility.await()
.untilAsserted(() -> assertThat(k8sclient.apps().deployments().withName(deploymentName).get()).isNotNull());
.untilAsserted(() -> assertThat(k8sclient.apps().statefulSets().withName(deploymentName).get()).isNotNull());
waitForKeycloakToBeReady(k8sclient, kc); // wait for reconciler to calm down to avoid race condititon
Log.info("Trying to modify deployment");
var deployment = k8sclient.apps().deployments().withName(deploymentName).get();
var deployment = k8sclient.apps().statefulSets().withName(deploymentName).get();
var labels = Map.of("address", "EvergreenTerrace742");
var flandersEnvVar = new EnvVarBuilder().withName("NEIGHBOR").withValue("Stupid Flanders!").build();
var origSpecs = new DeploymentSpecBuilder(deployment.getSpec()).build(); // deep copy
var origSpecs = new StatefulSetSpecBuilder(deployment.getSpec()).build(); // deep copy
deployment.getMetadata().getLabels().putAll(labels);
deployment.getSpec().getTemplate().getSpec().getContainers().get(0).setEnv(List.of(flandersEnvVar));
k8sclient.apps().deployments().createOrReplace(deployment);
k8sclient.apps().statefulSets().createOrReplace(deployment);
Awaitility.await()
.atMost(5, MINUTES)
.pollDelay(1, SECONDS)
.ignoreExceptions()
.untilAsserted(() -> {
var d = k8sclient.apps().deployments().withName(deploymentName).get();
var d = k8sclient.apps().statefulSets().withName(deploymentName).get();
assertThat(d.getMetadata().getLabels().entrySet().containsAll(labels.entrySet())).isTrue(); // additional labels should not be overwritten
assertThat(d.getSpec()).isEqualTo(origSpecs); // specs should be reconciled back to original values
});
@ -286,15 +286,36 @@ public class KeycloakDeploymentTest extends BaseOperatorTest {
@Test
public void testInitialAdminUser() {
try {
var kc = getDefaultKeycloakDeployment();
var kcAdminSecret = new KeycloakAdminSecret(k8sclient, kc);
k8sclient
.resources(Keycloak.class)
.inNamespace(namespace)
.delete();
k8sclient
.secrets()
.inNamespace(namespace)
.withName(kcAdminSecret.getName())
.delete();
// Making sure no other Keycloak pod is still around
Awaitility.await()
.ignoreExceptions()
.untilAsserted(() ->
assertThat(k8sclient
.pods()
.inNamespace(namespace)
.withLabel("app", "keycloak")
.list()
.getItems()
.size()).isZero());
// Recreating the database to keep this test isolated
deleteDB();
deployDB();
var kc = getDefaultKeycloakDeployment();
deployKeycloak(k8sclient, kc, true);
var decoder = Base64.getDecoder();
var service = new KeycloakService(k8sclient, kc);
var kcAdminSecret = new KeycloakAdminSecret(k8sclient, kc);
AtomicReference<String> adminUsername = new AtomicReference<>();
AtomicReference<String> adminPassword = new AtomicReference<>();

View file

@ -36,6 +36,7 @@ import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.assertj.core.api.Assertions.assertThat;
import static org.keycloak.operator.Constants.KEYCLOAK_HTTPS_PORT;
import static org.keycloak.operator.controllers.KeycloakDeployment.getEnvVarName;
import static org.keycloak.operator.testsuite.utils.K8sUtils.deployKeycloak;
import static org.keycloak.operator.testsuite.utils.K8sUtils.getDefaultKeycloakDeployment;
import static org.keycloak.operator.testsuite.utils.K8sUtils.inClusterCurl;
@ -115,6 +116,10 @@ public class RealmImportTest extends BaseOperatorTest {
CRAssert.assertKeycloakRealmImportStatusCondition(crSelector.get(), HAS_ERRORS, false);
});
var job = k8sclient.batch().v1().jobs().inNamespace(namespace).withName("example-count0-kc").get();
assertThat(job.getSpec().getTemplate().getMetadata().getLabels().get("app")).isEqualTo("keycloak-realm-import");
var envvars = job.getSpec().getTemplate().getSpec().getContainers().get(0).getEnv();
assertThat(envvars.stream().filter(e -> e.getName().equals(getEnvVarName("cache"))).findAny().get().getValue()).isEqualTo("local");
assertThat(envvars.stream().filter(e -> e.getName().equals(getEnvVarName("health-enabled"))).findAny().get().getValue()).isEqualTo("false");
assertThat(job.getSpec().getTemplate().getSpec().getImagePullSecrets().size()).isEqualTo(1);
assertThat(job.getSpec().getTemplate().getSpec().getImagePullSecrets().get(0).getName()).isEqualTo("my-empty-secret");

View file

@ -102,7 +102,21 @@ public class WatchedSecretsTest extends BaseOperatorTest {
Log.info("Checking pod logs for DB auth failures");
var podlogs = getPodNamesForCrs(Set.of(kc)).stream()
.filter(n -> !prevPodNames.contains(n)) // checking just new pods
.map(n -> k8sclient.pods().inNamespace(namespace).withName(n).getLog())
.map(n -> {
var name = k8sclient
.pods()
.inNamespace(namespace)
.list()
.getItems()
.stream()
.filter(p -> (p.getMetadata().getName() + p.getMetadata().getCreationTimestamp()).equals(n))
.findAny()
.get()
.getMetadata()
.getName();
return k8sclient.pods().inNamespace(namespace).withName(name).getLog();
})
.collect(Collectors.toList());
assertThat(podlogs).anyMatch(l -> l.contains("password authentication failed for user \"" + username + "\""));
});
@ -220,8 +234,13 @@ public class WatchedSecretsTest extends BaseOperatorTest {
}
private List<String> getPodNamesForCrs(Set<Keycloak> crs) {
return k8sclient.pods().inNamespace(namespace).list().getItems().stream()
.map(pod -> pod.getMetadata().getName())
return k8sclient
.pods()
.inNamespace(namespace)
.list()
.getItems()
.stream()
.map(pod -> pod.getMetadata().getName() + pod.getMetadata().getCreationTimestamp())
.filter(pod -> crs.stream().map(c -> c.getMetadata().getName()).anyMatch(pod::startsWith))
.collect(Collectors.toList());
}

View file

@ -21,7 +21,7 @@ import io.fabric8.kubernetes.api.model.IntOrString;
import io.fabric8.kubernetes.api.model.PodTemplateSpec;
import io.fabric8.kubernetes.api.model.PodTemplateSpecBuilder;
import io.fabric8.kubernetes.api.model.ProbeBuilder;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.api.model.apps.StatefulSet;
import io.quarkus.test.junit.QuarkusTest;
import org.junit.jupiter.api.Test;
import org.keycloak.operator.Config;
@ -36,7 +36,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
@QuarkusTest
public class PodTemplateTest {
Deployment getDeployment(PodTemplateSpec podTemplate) {
StatefulSet getDeployment(PodTemplateSpec podTemplate) {
var config = new Config(){
@Override
public Keycloak keycloak() {
@ -58,8 +58,8 @@ public class PodTemplateTest {
spec.setHostname("example.com");
spec.setTlsSecret("example-tls-secret");
kc.setSpec(spec);
var deployment = new KeycloakDeployment(null, config, kc, new Deployment(), "dummy-admin");
return (Deployment) deployment.getReconciledResource().get();
var deployment = new KeycloakDeployment(null, config, kc, new StatefulSet(), "dummy-admin");
return (StatefulSet) deployment.getReconciledResource().get();
}
@Test