fix: ensuring test state is clean between tests (#27904)

closes: #27080

Signed-off-by: Steve Hawkins <shawkins@redhat.com>

operator/src/test/java/org/keycloak/operator/testsuite/integration/BaseOperatorTest.java

@@ -22,8 +22,10 @@ import io.fabric8.kubernetes.api.model.MicroTime;
 import io.fabric8.kubernetes.api.model.NamespaceBuilder;
 import io.fabric8.kubernetes.api.model.Pod;
 import io.fabric8.kubernetes.api.model.Secret;
+import io.fabric8.kubernetes.api.model.Service;
 import io.fabric8.kubernetes.api.model.apps.Deployment;
 import io.fabric8.kubernetes.api.model.apps.StatefulSet;
+import io.fabric8.kubernetes.api.model.batch.v1.Job;
 import io.fabric8.kubernetes.api.model.events.v1.Event;
 import io.fabric8.kubernetes.api.model.rbac.ClusterRoleBinding;
 import io.fabric8.kubernetes.client.Config;
@@ -79,6 +81,7 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Optional;
 import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -286,19 +289,39 @@ public class BaseOperatorTest implements QuarkusTestAfterEachCallback {
   }
 
   public void cleanup() {
-    Log.info("Deleting Keycloak CR");
+      Log.info("Deleting Keycloak CR");
-    k8sclient.resources(Keycloak.class).delete();
+
-    Awaitility.await()
+      // due to https://github.com/operator-framework/java-operator-sdk/issues/2314 we
-            .untilAsserted(() -> {
+      // try to ensure that the operator has processed the delete event from root objects
-              var kcDeployments = k8sclient
+      // this can be simplified to just the root deletion after we pick up the fix
-                      .apps()
+      // it can be further simplified after https://github.com/fabric8io/kubernetes-client/issues/5838
-                      .statefulSets()
+      // to just a timed foreground deletion
-                      .inNamespace(namespace)
+      var roots = List.of(Keycloak.class, KeycloakRealmImport.class);
-                      .withLabels(Constants.DEFAULT_LABELS)
+      var dependents = List.of(StatefulSet.class, Secret.class, Service.class, Pod.class, Job.class);
-                      .list()
+
-                      .getItems();
+      var rootsDeleted = CompletableFuture.allOf(roots.stream()
-              assertThat(kcDeployments.size()).isZero();
+              .map(c -> k8sclient.resources(c).informOnCondition(List::isEmpty)).toArray(CompletableFuture[]::new));
-            });
+      roots.stream().forEach(c -> k8sclient.resources(c).withGracePeriod(0).delete());
+      try {
+          rootsDeleted.get(1, TimeUnit.MINUTES);
+      } catch (Exception e) {
+          // delete event should have arrived quickly because this is a background delete
+          throw new RuntimeException(e);
+      }
+      dependents.stream().map(c -> k8sclient.resources(c).withLabels(Constants.DEFAULT_LABELS))
+              .forEach(r -> r.withGracePeriod(0).delete());
+      // enforce that the dependents are gone
+      Awaitility.await().during(5, TimeUnit.SECONDS).until(() -> {
+          if (dependents.stream().anyMatch(
+                  c -> !k8sclient.resources(c).withLabels(Constants.DEFAULT_LABELS).list().getItems().isEmpty())) {
+              // the operator must have recreated because it hasn't gotten the keycloak
+              // deleted event, keep cleaning
+              dependents.stream().map(c -> k8sclient.resources(c).withLabels(Constants.DEFAULT_LABELS))
+                      .forEach(r -> r.withGracePeriod(0).delete());
+              return false;
+          }
+          return true;
+      });
   }
 
   @Override
@@ -316,6 +339,7 @@ public class BaseOperatorTest implements QuarkusTestAfterEachCallback {
               return;
           }
           Log.warnf("Test failed with %s: %s", context.getTestStatus().getTestErrorCause().getMessage(), context.getTestStatus().getTestErrorCause().getClass().getName());
+          Log.infof("Secrets %s", k8sclient.secrets().list().getItems().stream().map(s -> s.getMetadata().getName()).collect(Collectors.joining(", ")));
           logEvents();
           savePodLogs();
           // provide some helpful entries in the main log as well
@@ -325,7 +349,7 @@ public class BaseOperatorTest implements QuarkusTestAfterEachCallback {
           }
           logFailed(k8sclient.apps().statefulSets().withName(POSTGRESQL_NAME), StatefulSet::getStatus);
           k8sclient.pods().withLabel("app", "keycloak-realm-import").list().getItems().stream()
-                  .forEach(pod -> logFailed(k8sclient.pods().resource(pod), Pod::getStatus));
+                  .forEach(pod -> log(k8sclient.pods().resource(pod), Pod::getStatus, false));
       } finally {
           cleanup();
       }
@@ -339,11 +363,11 @@ public class BaseOperatorTest implements QuarkusTestAfterEachCallback {
           }
           Log.warnf("%s failed to become ready %s", instance.getMetadata().getName(), Serialization.asYaml(statusExtractor.apply(instance)));
       } else {
-          Log.infof("%s is ready %s", instance.getMetadata().getName(), Serialization.asYaml(statusExtractor.apply(instance)));
+          Log.infof("%s is ready %s %s", instance.getMetadata().getName(), resource.isReady(), Serialization.asYaml(statusExtractor.apply(instance)));
       }
       try {
           String log = resource.getLog();
-          log = log.substring(Math.max(0, log.length() - 5000));
+          log = log.substring(Math.max(0, log.length() - 50000));
           Log.warnf("%s log: %s", instance.getMetadata().getName(), log);
       } catch (KubernetesClientException e) {
           Log.warnf("No %s log: %s", instance.getMetadata().getName(), e.getMessage());

operator/src/test/java/org/keycloak/operator/testsuite/integration/PodTemplateTest.java

@@ -26,11 +26,12 @@ import io.fabric8.kubernetes.client.dsl.Resource;
 import io.fabric8.kubernetes.client.utils.Serialization;
 import io.quarkus.logging.Log;
 import io.quarkus.test.junit.QuarkusTest;
+
 import org.awaitility.Awaitility;
 import org.junit.jupiter.api.Test;
+import org.keycloak.operator.crds.v2alpha1.deployment.Keycloak;
 import org.keycloak.operator.testsuite.utils.CRAssert;
 import org.keycloak.operator.testsuite.utils.K8sUtils;
-import org.keycloak.operator.crds.v2alpha1.deployment.Keycloak;
 
 import java.util.Collections;
 
@@ -68,10 +69,7 @@ public class PodTemplateTest extends BaseOperatorTest {
             var keycloakPod = k8sclient
                     .pods()
                     .inNamespace(namespace)
-                    .withLabel("app", "keycloak")
+                    .withName("example-podtemplate-kc-0").get();
-                    .list()
-                    .getItems()
-                    .get(0);
 
             var logs = k8sclient
                     .pods()