From 26dc81a92fbce458d7c4b0cc7e33aef41c488ac7 Mon Sep 17 00:00:00 2001 From: Steven Hawkins Date: Wed, 27 Mar 2024 10:23:26 -0400 Subject: [PATCH] fix: ensuring test state is clean between tests (#27904) closes: #27080 Signed-off-by: Steve Hawkins --- .../integration/BaseOperatorTest.java | 56 +++++++++++++------ .../integration/PodTemplateTest.java | 8 +-- 2 files changed, 43 insertions(+), 21 deletions(-) diff --git a/operator/src/test/java/org/keycloak/operator/testsuite/integration/BaseOperatorTest.java b/operator/src/test/java/org/keycloak/operator/testsuite/integration/BaseOperatorTest.java index a49e53f557..41f8b8b0b4 100644 --- a/operator/src/test/java/org/keycloak/operator/testsuite/integration/BaseOperatorTest.java +++ b/operator/src/test/java/org/keycloak/operator/testsuite/integration/BaseOperatorTest.java @@ -22,8 +22,10 @@ import io.fabric8.kubernetes.api.model.MicroTime; import io.fabric8.kubernetes.api.model.NamespaceBuilder; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.Secret; +import io.fabric8.kubernetes.api.model.Service; import io.fabric8.kubernetes.api.model.apps.Deployment; import io.fabric8.kubernetes.api.model.apps.StatefulSet; +import io.fabric8.kubernetes.api.model.batch.v1.Job; import io.fabric8.kubernetes.api.model.events.v1.Event; import io.fabric8.kubernetes.api.model.rbac.ClusterRoleBinding; import io.fabric8.kubernetes.client.Config; @@ -79,6 +81,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; @@ -286,19 +289,39 @@ public class BaseOperatorTest implements QuarkusTestAfterEachCallback { } public void cleanup() { - Log.info("Deleting Keycloak CR"); - k8sclient.resources(Keycloak.class).delete(); - Awaitility.await() - .untilAsserted(() -> { - var kcDeployments = k8sclient - .apps() - .statefulSets() - .inNamespace(namespace) - .withLabels(Constants.DEFAULT_LABELS) - .list() - .getItems(); - assertThat(kcDeployments.size()).isZero(); - }); + Log.info("Deleting Keycloak CR"); + + // due to https://github.com/operator-framework/java-operator-sdk/issues/2314 we + // try to ensure that the operator has processed the delete event from root objects + // this can be simplified to just the root deletion after we pick up the fix + // it can be further simplified after https://github.com/fabric8io/kubernetes-client/issues/5838 + // to just a timed foreground deletion + var roots = List.of(Keycloak.class, KeycloakRealmImport.class); + var dependents = List.of(StatefulSet.class, Secret.class, Service.class, Pod.class, Job.class); + + var rootsDeleted = CompletableFuture.allOf(roots.stream() + .map(c -> k8sclient.resources(c).informOnCondition(List::isEmpty)).toArray(CompletableFuture[]::new)); + roots.stream().forEach(c -> k8sclient.resources(c).withGracePeriod(0).delete()); + try { + rootsDeleted.get(1, TimeUnit.MINUTES); + } catch (Exception e) { + // delete event should have arrived quickly because this is a background delete + throw new RuntimeException(e); + } + dependents.stream().map(c -> k8sclient.resources(c).withLabels(Constants.DEFAULT_LABELS)) + .forEach(r -> r.withGracePeriod(0).delete()); + // enforce that the dependents are gone + Awaitility.await().during(5, TimeUnit.SECONDS).until(() -> { + if (dependents.stream().anyMatch( + c -> !k8sclient.resources(c).withLabels(Constants.DEFAULT_LABELS).list().getItems().isEmpty())) { + // the operator must have recreated because it hasn't gotten the keycloak + // deleted event, keep cleaning + dependents.stream().map(c -> k8sclient.resources(c).withLabels(Constants.DEFAULT_LABELS)) + .forEach(r -> r.withGracePeriod(0).delete()); + return false; + } + return true; + }); } @Override @@ -316,6 +339,7 @@ public class BaseOperatorTest implements QuarkusTestAfterEachCallback { return; } Log.warnf("Test failed with %s: %s", context.getTestStatus().getTestErrorCause().getMessage(), context.getTestStatus().getTestErrorCause().getClass().getName()); + Log.infof("Secrets %s", k8sclient.secrets().list().getItems().stream().map(s -> s.getMetadata().getName()).collect(Collectors.joining(", "))); logEvents(); savePodLogs(); // provide some helpful entries in the main log as well @@ -325,7 +349,7 @@ public class BaseOperatorTest implements QuarkusTestAfterEachCallback { } logFailed(k8sclient.apps().statefulSets().withName(POSTGRESQL_NAME), StatefulSet::getStatus); k8sclient.pods().withLabel("app", "keycloak-realm-import").list().getItems().stream() - .forEach(pod -> logFailed(k8sclient.pods().resource(pod), Pod::getStatus)); + .forEach(pod -> log(k8sclient.pods().resource(pod), Pod::getStatus, false)); } finally { cleanup(); } @@ -339,11 +363,11 @@ public class BaseOperatorTest implements QuarkusTestAfterEachCallback { } Log.warnf("%s failed to become ready %s", instance.getMetadata().getName(), Serialization.asYaml(statusExtractor.apply(instance))); } else { - Log.infof("%s is ready %s", instance.getMetadata().getName(), Serialization.asYaml(statusExtractor.apply(instance))); + Log.infof("%s is ready %s %s", instance.getMetadata().getName(), resource.isReady(), Serialization.asYaml(statusExtractor.apply(instance))); } try { String log = resource.getLog(); - log = log.substring(Math.max(0, log.length() - 5000)); + log = log.substring(Math.max(0, log.length() - 50000)); Log.warnf("%s log: %s", instance.getMetadata().getName(), log); } catch (KubernetesClientException e) { Log.warnf("No %s log: %s", instance.getMetadata().getName(), e.getMessage()); diff --git a/operator/src/test/java/org/keycloak/operator/testsuite/integration/PodTemplateTest.java b/operator/src/test/java/org/keycloak/operator/testsuite/integration/PodTemplateTest.java index 40ba5adbe4..2323f178d5 100644 --- a/operator/src/test/java/org/keycloak/operator/testsuite/integration/PodTemplateTest.java +++ b/operator/src/test/java/org/keycloak/operator/testsuite/integration/PodTemplateTest.java @@ -26,11 +26,12 @@ import io.fabric8.kubernetes.client.dsl.Resource; import io.fabric8.kubernetes.client.utils.Serialization; import io.quarkus.logging.Log; import io.quarkus.test.junit.QuarkusTest; + import org.awaitility.Awaitility; import org.junit.jupiter.api.Test; +import org.keycloak.operator.crds.v2alpha1.deployment.Keycloak; import org.keycloak.operator.testsuite.utils.CRAssert; import org.keycloak.operator.testsuite.utils.K8sUtils; -import org.keycloak.operator.crds.v2alpha1.deployment.Keycloak; import java.util.Collections; @@ -68,10 +69,7 @@ public class PodTemplateTest extends BaseOperatorTest { var keycloakPod = k8sclient .pods() .inNamespace(namespace) - .withLabel("app", "keycloak") - .list() - .getItems() - .get(0); + .withName("example-podtemplate-kc-0").get(); var logs = k8sclient .pods()