operator bootstrap admin handling (#30711)

* enhance: add bootstrap admin handling to the operator

closes: #30004

Signed-off-by: Steve Hawkins <shawkins@redhat.com>

* Update docs/guides/operator/advanced-configuration.adoc

Co-authored-by: Václav Muzikář <vaclav@muzikari.cz>
Signed-off-by: Steven Hawkins <shawkins@redhat.com>

* enhance: add bootstrap admin handling to the operator

closes: #30004

Signed-off-by: Steve Hawkins <shawkins@redhat.com>

---------

Signed-off-by: Steve Hawkins <shawkins@redhat.com>
Signed-off-by: Steven Hawkins <shawkins@redhat.com>
Co-authored-by: Václav Muzikář <vaclav@muzikari.cz>

docs/guides/operator/advanced-configuration.adoc

@@ -308,4 +308,10 @@ stringData:
 When running on a Kubernetes or OpenShift environment well-known locations of trusted certificates are included automatically.
 This includes `/var/run/secrets/kubernetes.io/serviceaccount/ca.crt` and the `/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt` when present.
 
+=== Admin Bootstrapping
+
+When you create a new instance the Keycloak CR spec.bootstrapAdmin stanza may be used to configure the bootstrap user and/or service account. If you do not specify anything for the spec.bootstrapAdmin, the operator will create a Secret named "metadata.name"-initial-admin with a username temp-admin and a generated password. If you specify a Secret name for bootstrap admin user, then the Secret will need to contain `username` and `password` key value pairs. If you specify a Secret name for bootstrap admin service account, then the Secret will need to contain `client-id` and `client-secret` key value pairs.
+
+If a master realm has already been created for you cluster, then the spec.boostrapAdmin is effectively ignored. If you need to create a recovery admin account, then you'll need to run the CLI command against a Pod directly.
+
 </@tmpl.guide>

operator/src/main/java/org/keycloak/operator/controllers/KeycloakAdminSecretDependentResource.java

@@ -2,17 +2,21 @@ package org.keycloak.operator.controllers;
 
 import io.fabric8.kubernetes.api.model.Secret;
 import io.fabric8.kubernetes.api.model.SecretBuilder;
+import io.fabric8.kubernetes.api.model.networking.v1.Ingress;
 import io.fabric8.kubernetes.client.utils.KubernetesResourceUtil;
 import io.javaoperatorsdk.operator.api.reconciler.Context;
 import io.javaoperatorsdk.operator.api.reconciler.ResourceDiscriminator;
+import io.javaoperatorsdk.operator.api.reconciler.dependent.DependentResource;
 import io.javaoperatorsdk.operator.api.reconciler.dependent.GarbageCollected;
 import io.javaoperatorsdk.operator.processing.dependent.Creator;
 import io.javaoperatorsdk.operator.processing.dependent.kubernetes.KubernetesDependent;
 import io.javaoperatorsdk.operator.processing.dependent.kubernetes.KubernetesDependentResource;
+import io.javaoperatorsdk.operator.processing.dependent.workflow.Condition;
 
 import org.keycloak.operator.Constants;
 import org.keycloak.operator.Utils;
 import org.keycloak.operator.crds.v2alpha1.deployment.Keycloak;
+import org.keycloak.operator.crds.v2alpha1.deployment.spec.BootstrapAdminSpec;
 
 import java.util.Optional;
 import java.util.UUID;
@@ -20,6 +24,15 @@ import java.util.UUID;
 @KubernetesDependent(labelSelector = Constants.DEFAULT_LABELS_AS_STRING, resourceDiscriminator = KeycloakAdminSecretDependentResource.NameResourceDiscriminator.class)
 public class KeycloakAdminSecretDependentResource extends KubernetesDependentResource<Secret, Keycloak> implements Creator<Secret, Keycloak>, GarbageCollected<Keycloak> {
 
+    public static class EnabledCondition implements Condition<Ingress, Keycloak> {
+        @Override
+        public boolean isMet(DependentResource<Ingress, Keycloak> dependentResource, Keycloak primary,
+                Context<Keycloak> context) {
+            return Optional.ofNullable(primary.getSpec().getBootstrapAdminSpec()).map(BootstrapAdminSpec::getUser)
+                    .map(BootstrapAdminSpec.User::getSecret).filter(s -> !s.equals(KeycloakAdminSecretDependentResource.getName(primary))).isEmpty();
+        }
+    }
+
     public static class NameResourceDiscriminator implements ResourceDiscriminator<Secret, Keycloak> {
         @Override
         public Optional<Secret> distinguish(Class<Secret> resource, Keycloak primary, Context<Keycloak> context) {
@@ -39,8 +52,9 @@ public class KeycloakAdminSecretDependentResource extends KubernetesDependentRes
                 .addToLabels(Utils.allInstanceLabels(primary))
                 .withNamespace(primary.getMetadata().getNamespace())
                 .endMetadata()
+                .withType("Opaque")
                 .withType("kubernetes.io/basic-auth")
-                .addToData("username", Utils.asBase64("admin"))
+                .addToData("username", Utils.asBase64("temp-admin"))
                 .addToData("password", Utils.asBase64(UUID.randomUUID().toString().replace("-", "")))
                 .build();
     }

operator/src/main/java/org/keycloak/operator/controllers/KeycloakController.java

@@ -47,6 +47,7 @@ import org.keycloak.operator.Utils;
 import org.keycloak.operator.crds.v2alpha1.deployment.Keycloak;
 import org.keycloak.operator.crds.v2alpha1.deployment.KeycloakStatus;
 import org.keycloak.operator.crds.v2alpha1.deployment.KeycloakStatusAggregator;
+import org.keycloak.operator.crds.v2alpha1.deployment.spec.BootstrapAdminSpec;
 import org.keycloak.operator.crds.v2alpha1.deployment.spec.HostnameSpec;
 import org.keycloak.operator.crds.v2alpha1.deployment.spec.HostnameSpecBuilder;
 
@@ -61,7 +62,7 @@ import jakarta.inject.Inject;
 @ControllerConfiguration(
     dependents = {
         @Dependent(type = KeycloakDeploymentDependentResource.class),
-        @Dependent(type = KeycloakAdminSecretDependentResource.class),
+        @Dependent(type = KeycloakAdminSecretDependentResource.class, reconcilePrecondition = KeycloakAdminSecretDependentResource.EnabledCondition.class),
         @Dependent(type = KeycloakIngressDependentResource.class, reconcilePrecondition = KeycloakIngressDependentResource.EnabledCondition.class),
         @Dependent(type = KeycloakServiceDependentResource.class, useEventSourceWithName = "serviceSource"),
         @Dependent(type = KeycloakDiscoveryServiceDependentResource.class, useEventSourceWithName = "serviceSource")
@@ -104,6 +105,7 @@ public class KeycloakController implements Reconciler<Keycloak>, EventSourceInit
 
         Log.debugf("--- Reconciling Keycloak: %s in namespace: %s", kcName, namespace);
 
+        // TODO - these modifications to the resource belong in a webhook because dependents run first
         boolean modifiedSpec = false;
         if (kc.getSpec().getInstances() == null) {
             // explicitly set defaults - and let another reconciliation happen

operator/src/main/java/org/keycloak/operator/controllers/KeycloakDeploymentDependentResource.java

@@ -403,8 +403,7 @@ public class KeycloakDeploymentDependentResource extends CRUDKubernetesDependent
     private void addEnvVars(StatefulSet baseDeployment, Keycloak keycloakCR, TreeSet<String> allSecrets) {
         var firstClasssEnvVars = distConfigurator.configureDistOptions(keycloakCR);
 
-        String adminSecretName = KeycloakAdminSecretDependentResource.getName(keycloakCR);
+        var additionalEnvVars = getDefaultAndAdditionalEnvVars(keycloakCR);
-        var additionalEnvVars = getDefaultAndAdditionalEnvVars(keycloakCR, adminSecretName);
 
         var env = Optional.ofNullable(baseDeployment.getSpec().getTemplate().getSpec().getContainers().get(0).getEnv()).orElse(List.of());
 
@@ -426,15 +425,14 @@ public class KeycloakDeploymentDependentResource extends CRUDKubernetesDependent
 
         // watch the secrets used by secret key - we don't currently expect configmaps, optional refs, or watch the initial-admin
         TreeSet<String> serverConfigSecretsNames = envVars.stream().map(EnvVar::getValueFrom).filter(Objects::nonNull)
-                .map(EnvVarSource::getSecretKeyRef).filter(Objects::nonNull).map(SecretKeySelector::getName)
+                .map(EnvVarSource::getSecretKeyRef).filter(Objects::nonNull).map(SecretKeySelector::getName).collect(Collectors.toCollection(TreeSet::new));
-                .filter(n -> !n.equals(adminSecretName)).collect(Collectors.toCollection(TreeSet::new));
 
         Log.debugf("Found config secrets names: %s", serverConfigSecretsNames);
 
         allSecrets.addAll(serverConfigSecretsNames);
     }
 
-    private List<EnvVar> getDefaultAndAdditionalEnvVars(Keycloak keycloakCR, String adminSecretName) {
+    private List<EnvVar> getDefaultAndAdditionalEnvVars(Keycloak keycloakCR) {
         // default config values
         List<ValueOrSecret> serverConfigsList = new ArrayList<>(Constants.DEFAULT_DIST_CONFIG_LIST);
 
@@ -467,29 +465,6 @@ public class KeycloakDeploymentDependentResource extends CRUDKubernetesDependent
             }
         }
 
-        envVars.add(
-                new EnvVarBuilder()
-                        .withName("KEYCLOAK_ADMIN")
-                        .withNewValueFrom()
-                        .withNewSecretKeyRef()
-                        .withName(adminSecretName)
-                        .withKey("username")
-                        .withOptional(false)
-                        .endSecretKeyRef()
-                        .endValueFrom()
-                        .build());
-        envVars.add(
-                new EnvVarBuilder()
-                        .withName("KEYCLOAK_ADMIN_PASSWORD")
-                        .withNewValueFrom()
-                        .withNewSecretKeyRef()
-                        .withName(adminSecretName)
-                        .withKey("password")
-                        .withOptional(false)
-                        .endSecretKeyRef()
-                        .endValueFrom()
-                        .build());
-
         return envVars;
     }
 

operator/src/main/java/org/keycloak/operator/controllers/KeycloakDistConfigurator.java

@@ -28,6 +28,7 @@ import org.keycloak.operator.Constants;
 import org.keycloak.operator.crds.v2alpha1.deployment.Keycloak;
 import org.keycloak.operator.crds.v2alpha1.deployment.KeycloakStatusAggregator;
 import org.keycloak.operator.crds.v2alpha1.deployment.ValueOrSecret;
+import org.keycloak.operator.crds.v2alpha1.deployment.spec.BootstrapAdminSpec;
 import org.keycloak.operator.crds.v2alpha1.deployment.spec.DatabaseSpec;
 import org.keycloak.operator.crds.v2alpha1.deployment.spec.FeatureSpec;
 import org.keycloak.operator.crds.v2alpha1.deployment.spec.HostnameSpec;
@@ -73,6 +74,7 @@ public class KeycloakDistConfigurator {
         configureCache();
         configureProxy();
         configureManagement();
+        configureBootstrapAdmin();
     }
 
     /**
@@ -86,6 +88,26 @@ public class KeycloakDistConfigurator {
 
     /* ---------- Configuration of first-class citizen fields ---------- */
 
+    void configureBootstrapAdmin() {
+        optionMapper(Function.identity())
+                .mapOption("bootstrap-admin-username",
+                        keycloakCR -> Optional.ofNullable(keycloakCR.getSpec().getBootstrapAdminSpec())
+                                .map(BootstrapAdminSpec::getUser).map(BootstrapAdminSpec.User::getSecret)
+                                .or(() -> Optional.of(KeycloakAdminSecretDependentResource.getName(keycloakCR)))
+                                .map(s -> new SecretKeySelector("username", s, null)).orElse(null))
+                .mapOption("bootstrap-admin-password",
+                        keycloakCR -> Optional.ofNullable(keycloakCR.getSpec().getBootstrapAdminSpec())
+                                .map(BootstrapAdminSpec::getUser).map(BootstrapAdminSpec.User::getSecret)
+                                .or(() -> Optional.of(KeycloakAdminSecretDependentResource.getName(keycloakCR)))
+                                .map(s -> new SecretKeySelector("password", s, null)).orElse(null));
+
+        optionMapper(keycloakCR -> keycloakCR.getSpec().getBootstrapAdminSpec())
+                .mapOption("bootstrap-admin-client-id",
+                        spec -> Optional.ofNullable(spec.getService()).map(BootstrapAdminSpec.Service::getSecret).map(s -> new SecretKeySelector("client-id", s, null)).orElse(null))
+                .mapOption("bootstrap-admin-client-secret",
+                        spec -> Optional.ofNullable(spec.getService()).map(BootstrapAdminSpec.Service::getSecret).map(s -> new SecretKeySelector("client-secret", s, null)).orElse(null));
+    }
+
     void configureHostname() {
         optionMapper(keycloakCR -> keycloakCR.getSpec().getHostnameSpec())
                 .mapOption("hostname", HostnameSpec::getHostname)

operator/src/main/java/org/keycloak/operator/crds/v2alpha1/deployment/KeycloakSpec.java

@@ -20,6 +20,7 @@ import io.fabric8.kubernetes.api.model.LocalObjectReference;
 import io.fabric8.kubernetes.api.model.ResourceRequirements;
 import io.fabric8.kubernetes.model.annotation.SpecReplicas;
 
+import org.keycloak.operator.crds.v2alpha1.deployment.spec.BootstrapAdminSpec;
 import org.keycloak.operator.crds.v2alpha1.deployment.spec.CacheSpec;
 import org.keycloak.operator.crds.v2alpha1.deployment.spec.DatabaseSpec;
 import org.keycloak.operator.crds.v2alpha1.deployment.spec.FeatureSpec;
@@ -115,6 +116,10 @@ public class KeycloakSpec {
     @JsonPropertyDescription("In this section you can configure Keycloak's scheduling")
     private SchedulingSpec schedulingSpec;
 
+    @JsonProperty("bootstrapAdmin")
+    @JsonPropertyDescription("In this section you can configure Keycloak's bootstrap admin - will be used only for inital cluster creation.")
+    private BootstrapAdminSpec bootstrapAdminSpec;
+
     public HttpSpec getHttpSpec() {
         return httpSpec;
     }
@@ -264,4 +269,12 @@ public class KeycloakSpec {
     public void setSchedulingSpec(SchedulingSpec schedulingSpec) {
         this.schedulingSpec = schedulingSpec;
     }
+    
+    public BootstrapAdminSpec getBootstrapAdminSpec() {
+        return bootstrapAdminSpec;
+    }
+
+    public void setBootstrapAdminSpec(BootstrapAdminSpec bootstrapAdminSpec) {
+        this.bootstrapAdminSpec = bootstrapAdminSpec;
+    }
 }

operator/src/main/java/org/keycloak/operator/crds/v2alpha1/deployment/spec/BootstrapAdminSpec.java

@@ -0,0 +1,69 @@
+package org.keycloak.operator.crds.v2alpha1.deployment.spec;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonPropertyDescription;
+
+import io.sundr.builder.annotations.Buildable;
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@Buildable(editableEnabled = false, builderPackage = "io.fabric8.kubernetes.api.builder")
+public class BootstrapAdminSpec {
+
+    public static class User {
+        @JsonPropertyDescription("Name of the Secret that contains the username and password keys")
+    	private String secret;
+
+        public String getSecret() {
+            return secret;
+        }
+
+        public void setSecret(String secret) {
+            this.secret = secret;
+        }
+    }
+
+    public static class Service {
+        @JsonPropertyDescription("Name of the Secret that contains the client-id and client-secret keys")
+    	private String secret;
+
+        public String getSecret() {
+            return secret;
+        }
+
+        public void setSecret(String secret) {
+            this.secret = secret;
+        }
+
+    }
+
+    //private Integer expiration;
+    @JsonPropertyDescription("Configures the bootstrap admin user")
+    private User user;
+    @JsonPropertyDescription("Configures the bootstrap admin service account")
+    private Service service;
+
+    /*public Integer getExpiration() {
+        return expiration;
+    }
+
+    public void setExpiration(Integer expiration) {
+        this.expiration = expiration;
+    }*/
+
+    public User getUser() {
+        return user;
+    }
+
+    public void setUser(User user) {
+        this.user = user;
+    }
+
+    public Service getService() {
+        return service;
+    }
+
+    public void setService(Service service) {
+        this.service = service;
+    }
+
+}

operator/src/test/java/org/keycloak/operator/testsuite/integration/KeycloakDeploymentTest.java

@@ -46,6 +46,7 @@ import org.keycloak.operator.controllers.KeycloakServiceDependentResource;
 import org.keycloak.operator.crds.v2alpha1.deployment.Keycloak;
 import org.keycloak.operator.crds.v2alpha1.deployment.KeycloakStatusCondition;
 import org.keycloak.operator.crds.v2alpha1.deployment.ValueOrSecret;
+import org.keycloak.operator.crds.v2alpha1.deployment.spec.BootstrapAdminSpec;
 import org.keycloak.operator.crds.v2alpha1.deployment.spec.HostnameSpecBuilder;
 import org.keycloak.operator.testsuite.unit.WatchedResourcesTest;
 import org.keycloak.operator.testsuite.utils.CRAssert;
@@ -68,7 +69,6 @@ import static java.util.concurrent.TimeUnit.MINUTES;
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.keycloak.operator.testsuite.utils.CRAssert.assertKeycloakStatusCondition;
 import static org.keycloak.operator.testsuite.utils.K8sUtils.deployKeycloak;
@@ -384,22 +384,29 @@ public class KeycloakDeploymentTest extends BaseOperatorTest {
         assertKeycloakAccessibleViaService(kc, false, httpPort);
     }
     
-    // Reference curl command:
-    // curl --insecure --data "grant_type=password&client_id=admin-cli&username=admin&password=adminPassword" https://localhost:8443/realms/master/protocol/openid-connect/token
     @Test
     public void testInitialAdminUser() {
         var kc = getTestKeycloakDeployment(true);
         String secretName = KeycloakAdminSecretDependentResource.getName(kc);
+        assertInitialAdminUser(secretName, kc, false);
+    }
     
-        k8sclient
+    @Test
-                .resources(Keycloak.class)
+    public void testCustomBootstrapAdminUser() {
-                .inNamespace(namespace)
+        var kc = getTestKeycloakDeployment(true);
-                .delete();
+        String secretName = "my-secret";
-        k8sclient
+        // fluents don't seem to work here because of the inner classes
-                .secrets()
+        kc.getSpec().setBootstrapAdminSpec(new BootstrapAdminSpec());
-                .inNamespace(namespace)
+        kc.getSpec().getBootstrapAdminSpec().setUser(new BootstrapAdminSpec.User());
-                .withName(secretName)
+        kc.getSpec().getBootstrapAdminSpec().getUser().setSecret(secretName);
-                .delete();
+        k8sclient.resource(new SecretBuilder().withNewMetadata().withName(secretName).endMetadata()
+                .addToStringData("username", "user").addToStringData("password", "pass20rd").build()).create();
+        assertInitialAdminUser(secretName, kc, true);
+    }
+
+    // Reference curl command:
+    // curl --insecure --data "grant_type=password&client_id=admin-cli&username=admin&password=adminPassword" https://localhost:8443/realms/master/protocol/openid-connect/token
+    public void assertInitialAdminUser(String secretName, Keycloak kc, boolean samePasswordAfterReinstall) {
         
         // Making sure no other Keycloak pod is still around
         Awaitility.await()
@@ -422,6 +429,7 @@ public class KeycloakDeploymentTest extends BaseOperatorTest {
         AtomicReference<String> adminPassword = new AtomicReference<>();
         Awaitility.await()
                 .ignoreExceptions()
+                .atMost(3, TimeUnit.MINUTES)
                 .untilAsserted(() -> {
                     Log.info("Checking secret, ns: " + namespace + ", name: " + secretName);
                     var adminSecret = k8sclient
@@ -443,11 +451,12 @@ public class KeycloakDeploymentTest extends BaseOperatorTest {
                     assertTrue(curlOutput.contains("\"token_type\":\"Bearer\""));
                 });
 
-        // Redeploy the same Keycloak without redeploying the Database
+        // Redeploy the same Keycloak without redeploying the Database - the secret may change, but the admin password does not
         k8sclient.resource(kc).delete();
         deployKeycloak(k8sclient, kc, true);
         Awaitility.await()
                 .ignoreExceptions()
+                .atMost(3, TimeUnit.MINUTES)
                 .untilAsserted(() -> {
                     Log.info("Checking secret, ns: " + namespace + ", name: " + secretName);
                     var adminSecret = k8sclient
@@ -466,7 +475,7 @@ public class KeycloakDeploymentTest extends BaseOperatorTest {
 
                     assertTrue(curlOutput.contains("\"access_token\""));
                     assertTrue(curlOutput.contains("\"token_type\":\"Bearer\""));
-                    assertNotEquals(adminPassword.get(), newPassword);
+                    assertEquals(samePasswordAfterReinstall, adminPassword.get().equals(newPassword));
                 });
     }
 

operator/src/test/java/org/keycloak/operator/testsuite/unit/KeycloakDistConfiguratorTest.java

@@ -155,6 +155,18 @@ public class KeycloakDistConfiguratorTest {
         testFirstClassCitizen(expectedValues);
     }
     
+    @Test
+    public void bootstrapAdmin() {
+        final Map<String, String> expectedValues = Map.of(
+                "bootstrap-admin-username", "something",
+                "bootstrap-admin-password", "something",
+                "bootstrap-admin-client-id", "else",
+                "bootstrap-admin-client-secret", "else"
+        );
+
+        testFirstClassCitizen(expectedValues);
+    }
+
     /* UTILS */
 
     private void testFirstClassCitizen(Map<String, String> expectedValues) {

operator/src/test/resources/test-serialization-keycloak-cr.yml

@@ -72,6 +72,11 @@ spec:
         name: my-secret
   httpManagement:
     port: 9003
+  bootstrapAdmin:
+    user:
+      secret: something
+    service:
+      secret: else
   unsupported:
     podTemplate:
       metadata: