fix: ensuring test state is clean between tests (#27904)

closes: #27080

Signed-off-by: Steve Hawkins <shawkins@redhat.com>
This commit is contained in:
Steven Hawkins 2024-03-27 10:23:26 -04:00 committed by GitHub
parent 8407ae3bc5
commit 26dc81a92f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 43 additions and 21 deletions

View file

@ -22,8 +22,10 @@ import io.fabric8.kubernetes.api.model.MicroTime;
import io.fabric8.kubernetes.api.model.NamespaceBuilder;
import io.fabric8.kubernetes.api.model.Pod;
import io.fabric8.kubernetes.api.model.Secret;
import io.fabric8.kubernetes.api.model.Service;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.api.model.apps.StatefulSet;
import io.fabric8.kubernetes.api.model.batch.v1.Job;
import io.fabric8.kubernetes.api.model.events.v1.Event;
import io.fabric8.kubernetes.api.model.rbac.ClusterRoleBinding;
import io.fabric8.kubernetes.client.Config;
@ -79,6 +81,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.stream.Collectors;
@ -286,19 +289,39 @@ public class BaseOperatorTest implements QuarkusTestAfterEachCallback {
}
public void cleanup() {
Log.info("Deleting Keycloak CR");
k8sclient.resources(Keycloak.class).delete();
Awaitility.await()
.untilAsserted(() -> {
var kcDeployments = k8sclient
.apps()
.statefulSets()
.inNamespace(namespace)
.withLabels(Constants.DEFAULT_LABELS)
.list()
.getItems();
assertThat(kcDeployments.size()).isZero();
});
Log.info("Deleting Keycloak CR");
// due to https://github.com/operator-framework/java-operator-sdk/issues/2314 we
// try to ensure that the operator has processed the delete event from root objects
// this can be simplified to just the root deletion after we pick up the fix
// it can be further simplified after https://github.com/fabric8io/kubernetes-client/issues/5838
// to just a timed foreground deletion
var roots = List.of(Keycloak.class, KeycloakRealmImport.class);
var dependents = List.of(StatefulSet.class, Secret.class, Service.class, Pod.class, Job.class);
var rootsDeleted = CompletableFuture.allOf(roots.stream()
.map(c -> k8sclient.resources(c).informOnCondition(List::isEmpty)).toArray(CompletableFuture[]::new));
roots.stream().forEach(c -> k8sclient.resources(c).withGracePeriod(0).delete());
try {
rootsDeleted.get(1, TimeUnit.MINUTES);
} catch (Exception e) {
// delete event should have arrived quickly because this is a background delete
throw new RuntimeException(e);
}
dependents.stream().map(c -> k8sclient.resources(c).withLabels(Constants.DEFAULT_LABELS))
.forEach(r -> r.withGracePeriod(0).delete());
// enforce that the dependents are gone
Awaitility.await().during(5, TimeUnit.SECONDS).until(() -> {
if (dependents.stream().anyMatch(
c -> !k8sclient.resources(c).withLabels(Constants.DEFAULT_LABELS).list().getItems().isEmpty())) {
// the operator must have recreated because it hasn't gotten the keycloak
// deleted event, keep cleaning
dependents.stream().map(c -> k8sclient.resources(c).withLabels(Constants.DEFAULT_LABELS))
.forEach(r -> r.withGracePeriod(0).delete());
return false;
}
return true;
});
}
@Override
@ -316,6 +339,7 @@ public class BaseOperatorTest implements QuarkusTestAfterEachCallback {
return;
}
Log.warnf("Test failed with %s: %s", context.getTestStatus().getTestErrorCause().getMessage(), context.getTestStatus().getTestErrorCause().getClass().getName());
Log.infof("Secrets %s", k8sclient.secrets().list().getItems().stream().map(s -> s.getMetadata().getName()).collect(Collectors.joining(", ")));
logEvents();
savePodLogs();
// provide some helpful entries in the main log as well
@ -325,7 +349,7 @@ public class BaseOperatorTest implements QuarkusTestAfterEachCallback {
}
logFailed(k8sclient.apps().statefulSets().withName(POSTGRESQL_NAME), StatefulSet::getStatus);
k8sclient.pods().withLabel("app", "keycloak-realm-import").list().getItems().stream()
.forEach(pod -> logFailed(k8sclient.pods().resource(pod), Pod::getStatus));
.forEach(pod -> log(k8sclient.pods().resource(pod), Pod::getStatus, false));
} finally {
cleanup();
}
@ -339,11 +363,11 @@ public class BaseOperatorTest implements QuarkusTestAfterEachCallback {
}
Log.warnf("%s failed to become ready %s", instance.getMetadata().getName(), Serialization.asYaml(statusExtractor.apply(instance)));
} else {
Log.infof("%s is ready %s", instance.getMetadata().getName(), Serialization.asYaml(statusExtractor.apply(instance)));
Log.infof("%s is ready %s %s", instance.getMetadata().getName(), resource.isReady(), Serialization.asYaml(statusExtractor.apply(instance)));
}
try {
String log = resource.getLog();
log = log.substring(Math.max(0, log.length() - 5000));
log = log.substring(Math.max(0, log.length() - 50000));
Log.warnf("%s log: %s", instance.getMetadata().getName(), log);
} catch (KubernetesClientException e) {
Log.warnf("No %s log: %s", instance.getMetadata().getName(), e.getMessage());

View file

@ -26,11 +26,12 @@ import io.fabric8.kubernetes.client.dsl.Resource;
import io.fabric8.kubernetes.client.utils.Serialization;
import io.quarkus.logging.Log;
import io.quarkus.test.junit.QuarkusTest;
import org.awaitility.Awaitility;
import org.junit.jupiter.api.Test;
import org.keycloak.operator.crds.v2alpha1.deployment.Keycloak;
import org.keycloak.operator.testsuite.utils.CRAssert;
import org.keycloak.operator.testsuite.utils.K8sUtils;
import org.keycloak.operator.crds.v2alpha1.deployment.Keycloak;
import java.util.Collections;
@ -68,10 +69,7 @@ public class PodTemplateTest extends BaseOperatorTest {
var keycloakPod = k8sclient
.pods()
.inNamespace(namespace)
.withLabel("app", "keycloak")
.list()
.getItems()
.get(0);
.withName("example-podtemplate-kc-0").get();
var logs = k8sclient
.pods()