Test Baseline (#9625)

Closes #9174

Signed-off-by: jonathan <jvilalop@redhat.com>

.github/workflows/operator-ci.yml

@@ -34,6 +34,18 @@ jobs:
         run: |
           mvn clean install -DskipTests -DskipExamples -DskipTestsuite
 
-      - name: Build the Keycloak Operator
+      - name: Setup Minikube-Kubernetes
+        uses: manusa/actions-setup-minikube@v2.4.3
+        with:
+          minikube version: v1.24.0
+          kubernetes version: v1.22.3
+          github token: ${{ secrets.GITHUB_TOKEN }}
+          driver: docker
+      - name: Build , deploy and test operator in minikube
         run: |
-          mvn clean package -nsu -B -e -pl operator -Doperator -Dquarkus.container-image.build=true -Dquarkus.kubernetes.deployment-target=minikube
+          cd operator
+          eval $(minikube -p minikube docker-env) 
+          mvn clean verify \
+              -Dquarkus.container-image.build=true -Dquarkus.container-image.tag=test \
+              -Dquarkus.kubernetes.deployment-target=kubernetes \
+              --no-transfer-progress -Dtest.operator.deployment=remote

operator/pom.xml

@@ -32,11 +32,12 @@
         <maven.compiler.target>11</maven.compiler.target>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-        <quarkus.operator.sdk.version>3.0.0-SNAPSHOT</quarkus.operator.sdk.version>
+        <quarkus.operator.sdk.version>3.0.2</quarkus.operator.sdk.version>
         <quarkus.version>2.6.1.Final</quarkus.version>
         <quarkus.container-image.group>keycloak</quarkus.container-image.group>
         <quarkus.jib.base-jvm-image>eclipse-temurin:11</quarkus.jib.base-jvm-image>
         <quarkus.kubernetes.image-pull-policy>Never</quarkus.kubernetes.image-pull-policy>
+        <maven-failsafe-plugin.version>2.22.0</maven-failsafe-plugin.version>
     </properties>
     
     <repositories>
@@ -119,6 +120,30 @@
             <groupId>org.keycloak</groupId>
             <artifactId>keycloak-common</artifactId>
         </dependency>
+
+        <!-- Test -->
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-test-common</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-junit5</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.assertj</groupId>
+            <artifactId>assertj-core</artifactId>
+            <version>${assertj-core.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.awaitility</groupId>
+            <artifactId>awaitility</artifactId>
+            <version>${awaitility.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>
@@ -221,8 +246,19 @@
                         </configuration>
                     </execution>
                 </executions>
-              </plugin>
+            </plugin>
-
+            <plugin>
+                <artifactId>maven-failsafe-plugin</artifactId>
+                <version>${maven-failsafe-plugin.version}</version>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>integration-test</goal>
+                            <goal>verify</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
         </plugins>
     </build>
     

operator/src/main/java/org/keycloak/operator/Constants.java

@@ -33,6 +33,6 @@ public final class Constants {
     );
 
     public static final Map<String, String> DEFAULT_DIST_CONFIG = Map.of(
-        "KEYCLOAK_METRICS_ENABLED", "true"
+        "KC_METRICS_ENABLED", "true"
     );
 }

operator/src/main/java/org/keycloak/operator/v2alpha1/KeycloakController.java

@@ -18,7 +18,6 @@ package org.keycloak.operator.v2alpha1;
 
 import javax.inject.Inject;
 
-import io.fabric8.kubernetes.api.model.OwnerReference;
 import io.fabric8.kubernetes.api.model.apps.Deployment;
 import io.fabric8.kubernetes.client.KubernetesClient;
 import io.fabric8.kubernetes.client.informers.SharedIndexInformer;
@@ -30,9 +29,9 @@ import io.javaoperatorsdk.operator.api.reconciler.EventSourceInitializer;
 import io.javaoperatorsdk.operator.api.reconciler.Reconciler;
 import io.javaoperatorsdk.operator.api.reconciler.RetryInfo;
 import io.javaoperatorsdk.operator.api.reconciler.UpdateControl;
-import io.javaoperatorsdk.operator.processing.event.ResourceID;
 import io.javaoperatorsdk.operator.processing.event.source.EventSource;
 import io.javaoperatorsdk.operator.processing.event.source.informer.InformerEventSource;
+import io.javaoperatorsdk.operator.processing.event.source.informer.Mappers;
 import io.quarkus.logging.Log;
 import org.keycloak.operator.Config;
 import org.keycloak.operator.Constants;
@@ -43,11 +42,11 @@ import org.keycloak.operator.v2alpha1.crds.KeycloakStatusBuilder;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
-import java.util.Set;
 
+import static io.javaoperatorsdk.operator.api.reconciler.Constants.NO_FINALIZER;
 import static io.javaoperatorsdk.operator.api.reconciler.Constants.WATCH_CURRENT_NAMESPACE;
 
-@ControllerConfiguration(namespaces = WATCH_CURRENT_NAMESPACE)
+@ControllerConfiguration(namespaces = WATCH_CURRENT_NAMESPACE, finalizerName = NO_FINALIZER)
 public class KeycloakController implements Reconciler<Keycloak>, EventSourceInitializer<Keycloak>, ErrorStatusHandler<Keycloak> {
 
     @Inject
@@ -59,19 +58,11 @@ public class KeycloakController implements Reconciler<Keycloak>, EventSourceInit
     @Override
     public List<EventSource> prepareEventSources(EventSourceContext<Keycloak> context) {
         SharedIndexInformer<Deployment> deploymentInformer =
-                client.apps().deployments().inAnyNamespace()
+                client.apps().deployments().inNamespace(context.getConfigurationService().getClientConfiguration().getNamespace())
                         .withLabels(Constants.DEFAULT_LABELS)
                         .runnableInformer(0);
 
-        EventSource deploymentEvent = new InformerEventSource<>(
+        EventSource deploymentEvent = new InformerEventSource<>(deploymentInformer, Mappers.fromOwnerReference());
-                deploymentInformer, d -> {
-                    List<OwnerReference> ownerReferences = d.getMetadata().getOwnerReferences();
-                    if (!ownerReferences.isEmpty()) {
-                        return Set.of(new ResourceID(ownerReferences.get(0).getName(), d.getMetadata().getNamespace()));
-                    } else {
-                        return Collections.emptySet();
-                    }
-        });
 
         return List.of(deploymentEvent);
     }

operator/src/main/java/org/keycloak/operator/v2alpha1/crds/Keycloak.java

@@ -22,12 +22,20 @@ import io.fabric8.kubernetes.model.annotation.Group;
 import io.fabric8.kubernetes.model.annotation.Plural;
 import io.fabric8.kubernetes.model.annotation.ShortNames;
 import io.fabric8.kubernetes.model.annotation.Version;
+import io.sundr.builder.annotations.Buildable;
+import io.sundr.builder.annotations.BuildableReference;
 import org.keycloak.operator.Constants;
 
 @Group(Constants.CRDS_GROUP)
 @Version(Constants.CRDS_VERSION)
 @ShortNames(Constants.SHORT_NAME)
 @Plural(Constants.PLURAL_NAME)
+@Buildable(editableEnabled = false, builderPackage = "io.fabric8.kubernetes.api.builder",
+        lazyCollectionInitEnabled = false, refs = {
+        @BuildableReference(io.fabric8.kubernetes.api.model.ObjectMeta.class),
+        @BuildableReference(io.fabric8.kubernetes.client.CustomResource.class),
+        @BuildableReference(org.keycloak.operator.v2alpha1.crds.KeycloakSpec.class)
+})
 public class Keycloak extends CustomResource<KeycloakSpec, KeycloakStatus> implements Namespaced {
 
 }

operator/src/main/java/org/keycloak/operator/v2alpha1/crds/KeycloakSpec.java

@@ -16,8 +16,6 @@
  */
 package org.keycloak.operator.v2alpha1.crds;
 
-import io.fabric8.kubernetes.api.model.PodTemplate;
-
 import java.util.Map;
 
 public class KeycloakSpec {

operator/src/main/resources/base-keycloak-deployment.yaml

@@ -27,16 +27,19 @@ spec:
         - containerPort: 8080
           protocol: TCP
         livenessProbe:
-          exec:
+          httpGet:
-            command:
+            path: /health/live
-              - curl --head --fail --silent http://127.0.0.1:8080/health/live
+            port: 8080
-          periodSeconds: 1
+          initialDelaySeconds: 15
+          periodSeconds: 2
+          failureThreshold: 10
         readinessProbe:
-          exec:
+          httpGet:
-            command:
+            path: /health/ready
-              - curl --head --fail --silent http://127.0.0.1:8080/health/ready
+            port: 8080
-          periodSeconds: 1
+          initialDelaySeconds: 15
-          failureThreshold: 180
+          periodSeconds: 2
+          failureThreshold: 10
       dnsPolicy: ClusterFirst
       restartPolicy: Always
       terminationGracePeriodSeconds: 30

operator/src/main/resources/example-keycloak.yml

@@ -4,7 +4,7 @@ metadata:
   name: example-kc
 spec:
   instances: 1
-  distConfig:
+  serverConfiguration:
     KC_DB: postgres
     KC_DB_URL_HOST: postgres-db
 #    KC_DB_USERNAME: ${secret:keycloak-db-secret:username}

operator/src/test/java/org/keycloak/operator/ClusterOperatorTest.java

@@ -0,0 +1,142 @@
+package org.keycloak.operator;
+
+import io.fabric8.kubernetes.api.model.HasMetadata;
+import io.fabric8.kubernetes.api.model.NamespaceBuilder;
+import io.fabric8.kubernetes.api.model.apps.Deployment;
+import io.fabric8.kubernetes.client.Config;
+import io.fabric8.kubernetes.client.ConfigBuilder;
+import io.fabric8.kubernetes.client.DefaultKubernetesClient;
+import io.fabric8.kubernetes.client.KubernetesClient;
+import io.javaoperatorsdk.operator.Operator;
+import io.javaoperatorsdk.operator.api.reconciler.Reconciler;
+import io.quarkiverse.operatorsdk.runtime.OperatorProducer;
+import io.quarkiverse.operatorsdk.runtime.QuarkusConfigurationService;
+import io.quarkus.logging.Log;
+import org.eclipse.microprofile.config.ConfigProvider;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+
+import javax.enterprise.inject.Instance;
+import javax.enterprise.inject.spi.CDI;
+import javax.enterprise.util.TypeLiteral;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.util.List;
+import java.util.UUID;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public abstract class ClusterOperatorTest {
+
+  public static final String QUARKUS_KUBERNETES_DEPLOYMENT_TARGET = "quarkus.kubernetes.deployment-target";
+  public static final String OPERATOR_DEPLOYMENT_PROP = "test.operator.deployment";
+  public static final String TARGET_KUBERNETES_GENERATED_YML_FOLDER = "target/kubernetes/";
+
+  public enum OperatorDeployment {local,remote}
+
+  protected static OperatorDeployment operatorDeployment;
+  protected static Instance<Reconciler<? extends HasMetadata>> reconcilers;
+  protected static QuarkusConfigurationService configuration;
+  protected static KubernetesClient k8sclient;
+  protected static String namespace;
+  protected static String deploymentTarget;
+  private static Operator operator;
+
+
+  @BeforeAll
+  public static void before() throws FileNotFoundException {
+    configuration = CDI.current().select(QuarkusConfigurationService.class).get();
+    reconcilers = CDI.current().select(new TypeLiteral<>() {});
+    operatorDeployment = ConfigProvider.getConfig().getOptionalValue(OPERATOR_DEPLOYMENT_PROP, OperatorDeployment.class).orElse(OperatorDeployment.local);
+    deploymentTarget = ConfigProvider.getConfig().getOptionalValue(QUARKUS_KUBERNETES_DEPLOYMENT_TARGET, String.class).orElse("kubernetes");
+
+    calculateNamespace();
+    createK8sClient();
+    createNamespace();
+
+    if (operatorDeployment == OperatorDeployment.remote) {
+      createCRD();
+      createRBACresourcesAndOperatorDeployment();
+    } else {
+      createOperator();
+      registerReconcilers();
+      operator.start();
+    }
+
+  }
+
+  private static void createK8sClient() {
+    k8sclient = new DefaultKubernetesClient(new ConfigBuilder(Config.autoConfigure(null)).withNamespace(namespace).build());
+  }
+
+  private static void createRBACresourcesAndOperatorDeployment() throws FileNotFoundException {
+    Log.info("Creating RBAC into Namespace " + namespace);
+    List<HasMetadata> hasMetadata = k8sclient.load(new FileInputStream(TARGET_KUBERNETES_GENERATED_YML_FOLDER + deploymentTarget + ".yml"))
+            .inNamespace(namespace).get();
+    hasMetadata.stream()
+            .map(b -> {
+              if ("Deployment".equalsIgnoreCase(b.getKind()) && b.getMetadata().getName().contains("operator")) {
+                ((Deployment) b).getSpec().getTemplate().getSpec().getContainers().get(0).setImagePullPolicy("Never");
+              }
+              return b;
+            }).forEach(c -> {
+              Log.info("processing part : " + c.getKind() + "--" + c.getMetadata().getName() + " -- " + namespace);
+              k8sclient.resource(c).inNamespace(namespace).createOrReplace();
+            });
+  }
+
+  private static void cleanRBACresourcesAndOperatorDeployment() throws FileNotFoundException {
+    Log.info("Deleting RBAC from Namespace " + namespace);
+    k8sclient.load(new FileInputStream(TARGET_KUBERNETES_GENERATED_YML_FOLDER +deploymentTarget+".yml"))
+            .inNamespace(namespace).delete();
+  }
+  private static void createCRD() throws FileNotFoundException {
+    Log.info("Creating CRD ");
+    k8sclient.load(new FileInputStream(TARGET_KUBERNETES_GENERATED_YML_FOLDER + "keycloaks.keycloak.org-v1.yml")).createOrReplace();
+  }
+
+  private static void registerReconcilers() {
+    Log.info("Registering reconcilers for operator : " + operator + " [" + operatorDeployment + "]");
+
+    for (Reconciler reconciler : reconcilers) {
+      final var config = configuration.getConfigurationFor(reconciler);
+      if (!config.isRegistrationDelayed()) {
+        Log.info("Register and apply : " + reconciler.getClass().getName());
+        OperatorProducer.applyCRDIfNeededAndRegister(operator, reconciler, configuration);
+      }
+    }
+  }
+
+  private static void createOperator() {
+    operator = new Operator(k8sclient, configuration);
+    operator.getConfigurationService().getClientConfiguration().setNamespace(namespace);
+  }
+
+  private static void createNamespace() {
+    Log.info("Creating Namespace " + namespace);
+    k8sclient.namespaces().create(new NamespaceBuilder().withNewMetadata().withName(namespace).endMetadata().build());
+  }
+
+  private static void calculateNamespace() {
+    namespace = "keycloak-test-" + UUID.randomUUID();
+  }
+
+  @AfterAll
+  public static void after() throws FileNotFoundException {
+
+    if (operatorDeployment == OperatorDeployment.local) {
+      Log.info("Stopping Operator");
+      operator.stop();
+
+      Log.info("Creating new K8s Client");
+      // create a new client bc operator has closed the old one
+      createK8sClient();
+    } else {
+      cleanRBACresourcesAndOperatorDeployment();
+    }
+
+    Log.info("Deleting namespace : " + namespace);
+    assertThat(k8sclient.namespaces().withName(namespace).delete()).isTrue();
+    k8sclient.close();
+  }
+}

operator/src/test/java/org/keycloak/operator/OperatorE2EIT.java

@@ -0,0 +1,68 @@
+package org.keycloak.operator;
+
+import io.quarkus.logging.Log;
+import io.quarkus.test.junit.QuarkusTest;
+import org.awaitility.Awaitility;
+import org.awaitility.core.ConditionTimeoutException;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.time.Duration;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+@QuarkusTest
+public class OperatorE2EIT extends ClusterOperatorTest {
+    @Test
+    public void given_ClusterAndOperatorRunning_when_KeycloakCRCreated_Then_KeycloakStructureIsDeployedAndStatusIsOK() throws IOException {
+        Log.info(((operatorDeployment == OperatorDeployment.remote) ? "Remote " : "Local ") + "Run Test :" + namespace);
+
+        // DB
+        Log.info("Creating new PostgreSQL deployment");
+        k8sclient.load(OperatorE2EIT.class.getResourceAsStream("/example-postgres.yaml")).inNamespace(namespace).createOrReplace();
+
+        // Check DB has deployed and ready
+        Log.info("Checking Postgres is running");
+        Awaitility.await()
+                .atMost(Duration.ofSeconds(60))
+                .pollDelay(Duration.ofSeconds(2))
+                .untilAsserted(() -> assertThat(k8sclient.apps().statefulSets().inNamespace(namespace).withName("postgresql-db").get().getStatus().getReadyReplicas()).isEqualTo(1));
+        // CR
+        Log.info("Creating new Keycloak CR example");
+        k8sclient.load(OperatorE2EIT.class.getResourceAsStream("/example-keycloak.yml")).inNamespace(namespace).createOrReplace();
+
+        // Check Operator has deployed Keycloak
+        Log.info("Checking Operator has deployed Keycloak deployment");
+        Awaitility.await()
+                .atMost(Duration.ofSeconds(60))
+                .pollDelay(Duration.ofSeconds(2))
+                .untilAsserted(() -> assertThat(k8sclient.apps().deployments().inNamespace(namespace).withName("example-kc").get()).isNotNull());
+
+        // Check Keycloak has status ready
+        StringBuffer podlog = new StringBuffer();
+        try {
+            Log.info("Checking Keycloak pod has ready replicas == 1");
+            Awaitility.await()
+                    .atMost(Duration.ofSeconds(180))
+                    .pollDelay(Duration.ofSeconds(5))
+                    .untilAsserted(() -> {
+                        podlog.delete(0, podlog.length());
+                        try {
+                            k8sclient.pods().inNamespace(namespace).list().getItems().stream()
+                                    .filter(a -> a.getMetadata().getName().startsWith("example-kc"))
+                                    .forEach(a -> podlog.append(a.getMetadata().getName()).append(" : ")
+                                            .append(k8sclient.pods().inNamespace(namespace).withName(a.getMetadata().getName()).getLog(true)));
+                        } catch (Exception e) {
+                            // swallowing exception bc the pod is not ready to give logs yet
+                        }
+                        assertThat(k8sclient.apps().deployments().inNamespace(namespace).withName("example-kc").get().getStatus().getReadyReplicas()).isEqualTo(1);
+                    });
+        } catch (ConditionTimeoutException e) {
+            Log.error("On error POD LOG " + podlog, e);
+            throw e;
+        }
+
+
+    }
+
+}

pom.xml

@@ -158,6 +158,8 @@
         <selenium.version>2.35.0</selenium.version>
         <xml-apis.version>1.4.01</xml-apis.version>
         <subethasmtp.version>3.1.7</subethasmtp.version>
+        <awaitility.version>4.1.1</awaitility.version>
+        <assertj-core.version>3.22.0</assertj-core.version>
         <!-- KEYCLOAK-17585 Prevent microprofile-metrics-api upgrades from version "2.3" due to:
              https://issues.redhat.com/browse/KEYCLOAK-17585?focusedCommentId=16002705&page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#comment-16002705
         -->