From 819d33411acfcd4944190096cb8c357a26a859a8 Mon Sep 17 00:00:00 2001 From: Steven Hawkins Date: Thu, 27 Jul 2023 08:08:21 -0400 Subject: [PATCH] changes to speed up test times (#21658) Closes #10731 --- .../integration/BaseOperatorTest.java | 47 +++++++- .../testsuite/integration/ClusteringTest.java | 8 +- .../integration/KeycloakDeploymentTest.java | 43 ++++---- .../integration/KeycloakIngressTest.java | 17 ++- .../integration/KeycloakServicesTest.java | 4 +- .../integration/RealmImportTest.java | 42 ++++---- .../integration/WatchedSecretsTest.java | 101 ++++++++---------- .../testsuite/unit/PodTemplateTest.java | 15 ++- .../operator/testsuite/utils/K8sUtils.java | 79 +++++++------- 9 files changed, 200 insertions(+), 156 deletions(-) diff --git a/operator/src/test/java/org/keycloak/operator/testsuite/integration/BaseOperatorTest.java b/operator/src/test/java/org/keycloak/operator/testsuite/integration/BaseOperatorTest.java index 3c525133c8..b0e5f28e89 100644 --- a/operator/src/test/java/org/keycloak/operator/testsuite/integration/BaseOperatorTest.java +++ b/operator/src/test/java/org/keycloak/operator/testsuite/integration/BaseOperatorTest.java @@ -20,6 +20,8 @@ package org.keycloak.operator.testsuite.integration; import io.fabric8.kubernetes.api.model.HasMetadata; import io.fabric8.kubernetes.api.model.NamespaceBuilder; import io.fabric8.kubernetes.api.model.Secret; +import io.fabric8.kubernetes.api.model.PodSpecFluent.ContainersNested; +import io.fabric8.kubernetes.api.model.PodTemplateSpecFluent.SpecNested; import io.fabric8.kubernetes.client.Config; import io.fabric8.kubernetes.client.ConfigBuilder; import io.fabric8.kubernetes.client.KubernetesClient; @@ -41,6 +43,9 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.TestInfo; import org.keycloak.operator.Constants; import org.keycloak.operator.crds.v2alpha1.deployment.Keycloak; +import org.keycloak.operator.crds.v2alpha1.deployment.KeycloakSpecBuilder; +import org.keycloak.operator.crds.v2alpha1.deployment.KeycloakSpecFluent.UnsupportedNested; +import org.keycloak.operator.crds.v2alpha1.deployment.spec.UnsupportedSpecFluent.PodTemplateNested; import org.keycloak.operator.testsuite.utils.K8sUtils; import java.io.File; @@ -80,7 +85,6 @@ public abstract class BaseOperatorTest { private static Operator operator; protected static boolean isOpenShift; - @BeforeAll public static void before() throws FileNotFoundException { configuration = CDI.current().select(QuarkusConfigurationService.class).get(); @@ -263,4 +267,45 @@ public abstract class BaseOperatorTest { public static String getCurrentNamespace() { return namespace; } + + public static String getTestCustomImage() { + return customImage; + } + + /** + * Get the default deployment modified/optimized by operator test settings + * @param disableProbes when true the unsupported template will be used to effectively + * disable the probes, which will speed up testing for scenarios that don't interact + * with the underlying keycloak + * @return + */ + public static Keycloak getTestKeycloakDeployment(boolean disableProbes) { + Keycloak kc = K8sUtils.getDefaultKeycloakDeployment(); + kc.getMetadata().setNamespace(getCurrentNamespace()); + String image = getTestCustomImage(); + if (image != null) { + kc.getSpec().setImage(image); + } + if (disableProbes) { + return disableProbes(kc); + } + return kc; + } + + public static Keycloak disableProbes(Keycloak keycloak) { + KeycloakSpecBuilder specBuilder = new KeycloakSpecBuilder(keycloak.getSpec()); + var podTemplateSpecBuilder = specBuilder.editOrNewUnsupported().editOrNewPodTemplate().editOrNewSpec(); + ContainersNested>>> containerBuilder = null; + if (podTemplateSpecBuilder.hasContainers()) { + containerBuilder = podTemplateSpecBuilder.editContainer(0); + } else { + containerBuilder = podTemplateSpecBuilder.addNewContainer(); + } + keycloak.setSpec(containerBuilder.withNewLivenessProbe().withNewExec().addToCommand("true").endExec() + .endLivenessProbe().withNewReadinessProbe().withNewExec().addToCommand("true").endExec() + .endReadinessProbe().withNewStartupProbe().withNewExec().addToCommand("true").endExec() + .endStartupProbe().endContainer().endSpec().endPodTemplate().endUnsupported().build()); + return keycloak; + } + } diff --git a/operator/src/test/java/org/keycloak/operator/testsuite/integration/ClusteringTest.java b/operator/src/test/java/org/keycloak/operator/testsuite/integration/ClusteringTest.java index 70758abbd6..4bb8fc4673 100644 --- a/operator/src/test/java/org/keycloak/operator/testsuite/integration/ClusteringTest.java +++ b/operator/src/test/java/org/keycloak/operator/testsuite/integration/ClusteringTest.java @@ -54,11 +54,11 @@ public class ClusteringTest extends BaseOperatorTest { @Test public void testMultipleDeployments() throws InterruptedException { // given - var kc = K8sUtils.getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(true); // another instance running off the same database // - should eventually give this a separate schema - var kc1 = K8sUtils.getDefaultKeycloakDeployment(); + var kc1 = getTestKeycloakDeployment(true); kc1.getMetadata().setName("another-example"); kc1.getSpec().getHostnameSpec().setHostname("another-example.com"); // this is using the wrong tls-secret, but simply removing http spec renders the pod unstartable @@ -109,7 +109,7 @@ public class ClusteringTest extends BaseOperatorTest { @Test public void testKeycloakScaleAsExpected() { // given - var kc = K8sUtils.getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(false); var crSelector = k8sclient.resource(kc); K8sUtils.deployKeycloak(k8sclient, kc, true); @@ -196,7 +196,7 @@ public class ClusteringTest extends BaseOperatorTest { public void testKeycloakCacheIsConnected() throws Exception { // given Log.info("Setup"); - var kc = K8sUtils.getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(false); var crSelector = k8sclient.resource(kc); K8sUtils.deployKeycloak(k8sclient, kc, false); var targetInstances = 3; diff --git a/operator/src/test/java/org/keycloak/operator/testsuite/integration/KeycloakDeploymentTest.java b/operator/src/test/java/org/keycloak/operator/testsuite/integration/KeycloakDeploymentTest.java index 0ff70602fe..ae77162cea 100644 --- a/operator/src/test/java/org/keycloak/operator/testsuite/integration/KeycloakDeploymentTest.java +++ b/operator/src/test/java/org/keycloak/operator/testsuite/integration/KeycloakDeploymentTest.java @@ -62,7 +62,6 @@ import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.keycloak.operator.testsuite.utils.CRAssert.assertKeycloakStatusCondition; import static org.keycloak.operator.testsuite.utils.K8sUtils.deployKeycloak; -import static org.keycloak.operator.testsuite.utils.K8sUtils.getDefaultKeycloakDeployment; import static org.keycloak.operator.testsuite.utils.K8sUtils.getResourceFromFile; import static org.keycloak.operator.testsuite.utils.K8sUtils.waitForKeycloakToBeReady; @@ -73,7 +72,7 @@ public class KeycloakDeploymentTest extends BaseOperatorTest { try { // CR Log.info("Creating new Keycloak CR example"); - var kc = getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(true); var deploymentName = kc.getMetadata().getName(); deployKeycloak(k8sclient, kc, true); @@ -103,7 +102,7 @@ public class KeycloakDeploymentTest extends BaseOperatorTest { @Test public void testCRFields() { try { - var kc = getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(true); var deploymentName = kc.getMetadata().getName(); deployKeycloak(k8sclient, kc, true); @@ -137,7 +136,7 @@ public class KeycloakDeploymentTest extends BaseOperatorTest { @Test public void testConfigInCRTakesPrecedence() { try { - var defaultKCDeploy = getDefaultKeycloakDeployment(); + var defaultKCDeploy = getTestKeycloakDeployment(true); var valueSecretHealthProp = new ValueOrSecret("health-enabled", "false"); var valueSecretProxyProp = new ValueOrSecret("proxy", "reencrypt"); @@ -209,7 +208,7 @@ public class KeycloakDeploymentTest extends BaseOperatorTest { @Test public void testDeploymentDurability() { try { - var kc = getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(true); var deploymentName = kc.getMetadata().getName(); // create a dummy StatefulSet representing the pre-multiinstance state that we'll be forced to delete @@ -274,7 +273,7 @@ public class KeycloakDeploymentTest extends BaseOperatorTest { @Test public void testTlsUsesCorrectSecret() { try { - var kc = getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(true); deployKeycloak(k8sclient, kc, true); var service = new KeycloakService(k8sclient, kc); @@ -284,10 +283,10 @@ public class KeycloakDeploymentTest extends BaseOperatorTest { String url = "https://" + service.getName() + "." + namespace + ":" + Constants.KEYCLOAK_HTTPS_PORT; Log.info("Checking url: " + url); - var curlOutput = K8sUtils.inClusterCurl(k8sclient, namespace, "--insecure", "-s", "-v", url); + var curlOutput = K8sUtils.inClusterCurl(k8sclient, namespace, "--insecure", "-s", "-w", "%{certs}", url); Log.info("Curl Output: " + curlOutput); - assertTrue(curlOutput.contains("issuer: O=mkcert development CA; OU=aperuffo@aperuffo-mac (Andrea Peruffo); CN=mkcert aperuffo@aperuffo-mac (Andrea Peruffo)")); + assertTrue(curlOutput.contains("Issuer:O = mkcert development CA, OU = aperuffo@aperuffo-mac (Andrea Peruffo), CN = mkcert aperuffo@aperuffo-mac (Andrea Peruffo)")); }); } catch (Exception e) { savePodLogs(); @@ -298,7 +297,7 @@ public class KeycloakDeploymentTest extends BaseOperatorTest { @Test public void testTlsDisabled() { try { - var kc = getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(true); kc.getSpec().getHttpSpec().setTlsSecret(null); kc.getSpec().getHttpSpec().setHttpEnabled(true); deployKeycloak(k8sclient, kc, true); @@ -313,7 +312,7 @@ public class KeycloakDeploymentTest extends BaseOperatorTest { @Test public void testHostnameStrict() { try { - var kc = getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(true); deployKeycloak(k8sclient, kc, true); var service = new KeycloakService(k8sclient, kc); @@ -337,7 +336,7 @@ public class KeycloakDeploymentTest extends BaseOperatorTest { @Test public void testHostnameStrictDisabled() { try { - var kc = getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(true); var hostnameSpec = new HostnameSpecBuilder() .withStrict(false) .withStrictBackchannel(false) @@ -369,7 +368,7 @@ public class KeycloakDeploymentTest extends BaseOperatorTest { try { final int httpsPort = 8543; final int httpPort = 8180; - var kc = getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(true); kc.getSpec().getHttpSpec().setHttpsPort(httpsPort); kc.getSpec().getHttpSpec().setHttpPort(httpPort); @@ -393,7 +392,7 @@ public class KeycloakDeploymentTest extends BaseOperatorTest { try { final int httpsPort = 8543; final int httpPort = 8180; - var kc = getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(true); kc.getSpec().getHttpSpec().setHttpsPort(httpsPort); kc.getSpec().getHttpSpec().setHttpPort(httpPort); kc.getSpec().getHttpSpec().setTlsSecret(null); @@ -419,7 +418,7 @@ public class KeycloakDeploymentTest extends BaseOperatorTest { @Test public void testInitialAdminUser() { try { - var kc = getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(true); var kcAdminSecret = new KeycloakAdminSecret(k8sclient, kc); k8sclient @@ -510,7 +509,7 @@ public class KeycloakDeploymentTest extends BaseOperatorTest { @EnabledIfSystemProperty(named = OPERATOR_CUSTOM_IMAGE, matches = ".+") public void testCustomImage() { try { - var kc = getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(true); kc.getSpec().setImage(customImage); deployKeycloak(k8sclient, kc, true); @@ -535,7 +534,7 @@ public class KeycloakDeploymentTest extends BaseOperatorTest { String secretDescriptorFilename = "test-docker-registry-secret.yaml"; try { - var kc = getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(true); kc.getSpec().setImage(customImage); handleFakeImagePullSecretCreation(kc, secretDescriptorFilename); @@ -563,7 +562,7 @@ public class KeycloakDeploymentTest extends BaseOperatorTest { public void testInvalidCustomImageHasErrorMessage() { try { - var kc = getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(true); kc.getSpec().setImage("does-not-exist"); deployKeycloak(k8sclient, kc, false); @@ -587,7 +586,8 @@ public class KeycloakDeploymentTest extends BaseOperatorTest { @Test public void testHttpRelativePathWithPlainValue() { try { - var kc = getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(false); + kc.getSpec().setImage(null); // doesn't seem to become ready with the custom image kc.getSpec().getAdditionalOptions().add(new ValueOrSecret(Constants.KEYCLOAK_HTTP_RELATIVE_PATH_KEY, "/foobar")); deployKeycloak(k8sclient, kc, true); @@ -608,7 +608,8 @@ public class KeycloakDeploymentTest extends BaseOperatorTest { @Test public void testHttpRelativePathWithSecretValue() { try { - var kc = getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(false); + kc.getSpec().setImage(null); // doesn't seem to become ready with the custom image var secretName = "my-http-relative-path"; var keyName = "rel-path"; var httpRelativePathSecret = new SecretBuilder() @@ -644,7 +645,7 @@ public class KeycloakDeploymentTest extends BaseOperatorTest { @Test public void testUpgradeRecreatesPods() { try { - var kc = getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(true); kc.getSpec().setInstances(3); deployKeycloak(k8sclient, kc, true); @@ -691,7 +692,7 @@ public class KeycloakDeploymentTest extends BaseOperatorTest { "Skipping the test when Operator deployed remotely to keep stuff simple, it's just SmallRye, we don't need to retest it"); try { - var kc = getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(true); deployKeycloak(k8sclient, kc, true); // labels are set in test/resources/application.properties diff --git a/operator/src/test/java/org/keycloak/operator/testsuite/integration/KeycloakIngressTest.java b/operator/src/test/java/org/keycloak/operator/testsuite/integration/KeycloakIngressTest.java index 0888f06bb9..6d5282d67f 100644 --- a/operator/src/test/java/org/keycloak/operator/testsuite/integration/KeycloakIngressTest.java +++ b/operator/src/test/java/org/keycloak/operator/testsuite/integration/KeycloakIngressTest.java @@ -63,7 +63,7 @@ public class KeycloakIngressTest extends BaseOperatorTest { @Test public void testIngressOnHTTP() { - var kc = K8sUtils.getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(false); kc.getSpec().getHttpSpec().setTlsSecret(null); kc.getSpec().getHttpSpec().setHttpEnabled(true); var hostnameSpecBuilder = new HostnameSpecBuilder() @@ -89,7 +89,7 @@ public class KeycloakIngressTest extends BaseOperatorTest { @Test public void testIngressOnHTTPS() { - var kc = K8sUtils.getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(false); var hostnameSpecBuilder = new HostnameSpecBuilder() .withStrict(false) .withStrictBackchannel(false); @@ -142,7 +142,7 @@ public class KeycloakIngressTest extends BaseOperatorTest { @Test public void testIngressHostname() { - var kc = K8sUtils.getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(true); var hostnameSpec = new HostnameSpecBuilder().withHostname("foo.bar").build(); kc.getSpec().setHostnameSpec(hostnameSpec); @@ -170,7 +170,7 @@ public class KeycloakIngressTest extends BaseOperatorTest { @Test public void testMainIngressDurability() { - var kc = K8sUtils.getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(true); kc.getSpec().setIngressSpec(new IngressSpec()); kc.getSpec().getIngressSpec().setIngressEnabled(true); kc.getSpec().getIngressSpec().setAnnotations(Map.of("haproxy.router.openshift.io/disable_cookies", "true")); @@ -226,8 +226,7 @@ public class KeycloakIngressTest extends BaseOperatorTest { @Test public void testCustomIngressDeletion() { - - Keycloak defaultKeycloakDeployment = K8sUtils.getDefaultKeycloakDeployment(); + Keycloak defaultKeycloakDeployment = getTestKeycloakDeployment(true); String kcDeploymentName = defaultKeycloakDeployment.getMetadata().getName(); Resource customIngressDeployedManuallySelector = null; Ingress customIngressCreatedManually; @@ -271,10 +270,10 @@ public class KeycloakIngressTest extends BaseOperatorTest { } } } - + @Test public void testCustomIngressClassName() { - var kc = K8sUtils.getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(true); kc.getSpec().setIngressSpec(new IngressSpecBuilder().withIngressClassName("nginx").build()); K8sUtils.deployKeycloak(k8sclient, kc, true); @@ -307,7 +306,7 @@ public class KeycloakIngressTest extends BaseOperatorTest { @Test public void testCustomIngressAnnotations() { - var kc = K8sUtils.getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(true); kc.getSpec().setIngressSpec(new IngressSpec()); kc.getSpec().getIngressSpec().setIngressEnabled(true); diff --git a/operator/src/test/java/org/keycloak/operator/testsuite/integration/KeycloakServicesTest.java b/operator/src/test/java/org/keycloak/operator/testsuite/integration/KeycloakServicesTest.java index 120a9194d0..e1440c770d 100644 --- a/operator/src/test/java/org/keycloak/operator/testsuite/integration/KeycloakServicesTest.java +++ b/operator/src/test/java/org/keycloak/operator/testsuite/integration/KeycloakServicesTest.java @@ -35,7 +35,7 @@ import static org.assertj.core.api.Assertions.assertThat; public class KeycloakServicesTest extends BaseOperatorTest { @Test public void testMainServiceDurability() { - var kc = K8sUtils.getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(true); K8sUtils.deployKeycloak(k8sclient, kc, true); var service = new KeycloakService(k8sclient, kc); var serviceSelector = k8sclient.services().inNamespace(namespace).withName(service.getName()); @@ -84,7 +84,7 @@ public class KeycloakServicesTest extends BaseOperatorTest { @Test public void testDiscoveryServiceDurability() { - var kc = K8sUtils.getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(true); K8sUtils.deployKeycloak(k8sclient, kc, true); var discoveryService = new KeycloakDiscoveryService(k8sclient, kc); var discoveryServiceSelector = k8sclient.services().inNamespace(namespace).withName(discoveryService.getName()); diff --git a/operator/src/test/java/org/keycloak/operator/testsuite/integration/RealmImportTest.java b/operator/src/test/java/org/keycloak/operator/testsuite/integration/RealmImportTest.java index 4299213d94..eea22d3379 100644 --- a/operator/src/test/java/org/keycloak/operator/testsuite/integration/RealmImportTest.java +++ b/operator/src/test/java/org/keycloak/operator/testsuite/integration/RealmImportTest.java @@ -40,7 +40,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.keycloak.operator.Constants.KEYCLOAK_HTTPS_PORT; import static org.keycloak.operator.controllers.KeycloakDistConfigurator.getKeycloakOptionEnvVarName; import static org.keycloak.operator.testsuite.utils.K8sUtils.deployKeycloak; -import static org.keycloak.operator.testsuite.utils.K8sUtils.getDefaultKeycloakDeployment; import static org.keycloak.operator.testsuite.utils.K8sUtils.inClusterCurl; import static org.keycloak.operator.crds.v2alpha1.realmimport.KeycloakRealmImportStatusCondition.DONE; import static org.keycloak.operator.crds.v2alpha1.realmimport.KeycloakRealmImportStatusCondition.HAS_ERRORS; @@ -81,7 +80,8 @@ public class RealmImportTest extends BaseOperatorTest { @Test public void testWorkingRealmImport() { // Arrange - var kc = getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(false); + kc.getSpec().setImage(null); // checks the job args for the base, not custom image kc.getSpec().setImagePullSecrets(Arrays.asList(new LocalObjectReferenceBuilder().withName("my-empty-secret").build())); deployKeycloak(k8sclient, kc, false); @@ -98,9 +98,10 @@ public class RealmImportTest extends BaseOperatorTest { .pollDelay(1, SECONDS) .ignoreExceptions() .untilAsserted(() -> { - CRAssert.assertKeycloakRealmImportStatusCondition(crSelector.get(), DONE, false); - CRAssert.assertKeycloakRealmImportStatusCondition(crSelector.get(), STARTED, true); - CRAssert.assertKeycloakRealmImportStatusCondition(crSelector.get(), HAS_ERRORS, false); + KeycloakRealmImport cr = crSelector.get(); + CRAssert.assertKeycloakRealmImportStatusCondition(cr, DONE, false); + CRAssert.assertKeycloakRealmImportStatusCondition(cr, STARTED, true); + CRAssert.assertKeycloakRealmImportStatusCondition(cr, HAS_ERRORS, false); }); Awaitility.await() @@ -108,9 +109,10 @@ public class RealmImportTest extends BaseOperatorTest { .pollDelay(1, SECONDS) .ignoreExceptions() .untilAsserted(() -> { - CRAssert.assertKeycloakRealmImportStatusCondition(crSelector.get(), DONE, true); - CRAssert.assertKeycloakRealmImportStatusCondition(crSelector.get(), STARTED, false); - CRAssert.assertKeycloakRealmImportStatusCondition(crSelector.get(), HAS_ERRORS, false); + KeycloakRealmImport cr = crSelector.get(); + CRAssert.assertKeycloakRealmImportStatusCondition(cr, DONE, true); + CRAssert.assertKeycloakRealmImportStatusCondition(cr, STARTED, false); + CRAssert.assertKeycloakRealmImportStatusCondition(cr, HAS_ERRORS, false); }); var job = k8sclient.batch().v1().jobs().inNamespace(namespace).withName("example-count0-kc").get(); assertThat(job.getSpec().getTemplate().getMetadata().getLabels().get("app")).isEqualTo("keycloak-realm-import"); @@ -120,11 +122,10 @@ public class RealmImportTest extends BaseOperatorTest { assertThat(job.getSpec().getTemplate().getSpec().getImagePullSecrets().size()).isEqualTo(1); assertThat(job.getSpec().getTemplate().getSpec().getImagePullSecrets().get(0).getName()).isEqualTo("my-empty-secret"); - var service = new KeycloakService(k8sclient, getDefaultKeycloakDeployment()); String url = - "https://" + service.getName() + "." + namespace + ":" + KEYCLOAK_HTTPS_PORT + "/realms/count0"; + "https://" + KeycloakService.getServiceName(kc) + "." + namespace + ":" + KEYCLOAK_HTTPS_PORT + "/realms/count0"; - Awaitility.await().atMost(10, MINUTES).untilAsserted(() -> { + Awaitility.await().atMost(10, MINUTES).ignoreExceptions().untilAsserted(() -> { Log.info("Starting curl Pod to test if the realm is available"); Log.info("Url: '" + url + "'"); String curlOutput = inClusterCurl(k8sclient, namespace, url); @@ -139,7 +140,7 @@ public class RealmImportTest extends BaseOperatorTest { @EnabledIfSystemProperty(named = OPERATOR_CUSTOM_IMAGE, matches = ".+") public void testWorkingRealmImportWithCustomImage() { // Arrange - var keycloak = getDefaultKeycloakDeployment(); + var keycloak = getTestKeycloakDeployment(false); keycloak.getSpec().setImage(customImage); // Removing the Database so that a subsequent build will by default act on h2 // TODO: uncomment the following line after resolution of: https://github.com/keycloak/keycloak/issues/11767 @@ -160,9 +161,10 @@ public class RealmImportTest extends BaseOperatorTest { .pollDelay(5, SECONDS) .ignoreExceptions() .untilAsserted(() -> { - CRAssert.assertKeycloakRealmImportStatusCondition(crSelector.get(), DONE, true); - CRAssert.assertKeycloakRealmImportStatusCondition(crSelector.get(), STARTED, false); - CRAssert.assertKeycloakRealmImportStatusCondition(crSelector.get(), HAS_ERRORS, false); + KeycloakRealmImport cr = crSelector.get(); + CRAssert.assertKeycloakRealmImportStatusCondition(cr, DONE, true); + CRAssert.assertKeycloakRealmImportStatusCondition(cr, STARTED, false); + CRAssert.assertKeycloakRealmImportStatusCondition(cr, HAS_ERRORS, false); }); assertThat(getJobArgs()).doesNotContain("build"); @@ -171,7 +173,7 @@ public class RealmImportTest extends BaseOperatorTest { @Test public void testNotWorkingRealmImport() { // Arrange - deployKeycloak(k8sclient, getDefaultKeycloakDeployment(), true); // make sure there are no errors due to missing KC Deployment + deployKeycloak(k8sclient, getTestKeycloakDeployment(false), true); // make sure there are no errors due to missing KC Deployment // Act K8sUtils.set(k8sclient, getClass().getResourceAsStream("/incorrect-realm.yaml")); @@ -186,10 +188,10 @@ public class RealmImportTest extends BaseOperatorTest { .resources(KeycloakRealmImport.class) .inNamespace(namespace) .withName("example-count0-kc"); - - CRAssert.assertKeycloakRealmImportStatusCondition(crSelector.get(), DONE, false); - CRAssert.assertKeycloakRealmImportStatusCondition(crSelector.get(), STARTED, false); - CRAssert.assertKeycloakRealmImportStatusCondition(crSelector.get(), HAS_ERRORS, true); + KeycloakRealmImport cr = crSelector.get(); + CRAssert.assertKeycloakRealmImportStatusCondition(cr, DONE, false); + CRAssert.assertKeycloakRealmImportStatusCondition(cr, STARTED, false); + CRAssert.assertKeycloakRealmImportStatusCondition(cr, HAS_ERRORS, true); }); } diff --git a/operator/src/test/java/org/keycloak/operator/testsuite/integration/WatchedSecretsTest.java b/operator/src/test/java/org/keycloak/operator/testsuite/integration/WatchedSecretsTest.java index b3e427bd8d..012a1e0844 100644 --- a/operator/src/test/java/org/keycloak/operator/testsuite/integration/WatchedSecretsTest.java +++ b/operator/src/test/java/org/keycloak/operator/testsuite/integration/WatchedSecretsTest.java @@ -17,10 +17,13 @@ package org.keycloak.operator.testsuite.integration; +import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.Secret; import io.fabric8.kubernetes.api.model.SecretBuilder; +import io.fabric8.kubernetes.api.model.apps.StatefulSet; import io.quarkus.logging.Log; import io.quarkus.test.junit.QuarkusTest; + import org.awaitility.Awaitility; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; @@ -31,18 +34,19 @@ import org.keycloak.operator.crds.v2alpha1.deployment.KeycloakStatusCondition; import org.keycloak.operator.crds.v2alpha1.deployment.ValueOrSecret; import org.keycloak.operator.crds.v2alpha1.deployment.spec.HostnameSpecBuilder; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.HashSet; -import java.util.List; import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; import static org.keycloak.operator.testsuite.utils.CRAssert.assertKeycloakStatusCondition; import static org.keycloak.operator.testsuite.utils.K8sUtils.deployKeycloak; -import static org.keycloak.operator.testsuite.utils.K8sUtils.getDefaultKeycloakDeployment; /** * @author Vaclav Muzikar @@ -52,7 +56,7 @@ public class WatchedSecretsTest extends BaseOperatorTest { @Test public void testSecretsAreWatched() { try { - var kc = getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(true); deployKeycloak(k8sclient, kc, true); Secret dbSecret = getDbSecret(); @@ -89,10 +93,10 @@ public class WatchedSecretsTest extends BaseOperatorTest { try { final String username = "HomerSimpson"; - var kc = getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(false); deployKeycloak(k8sclient, kc, true); - var prevPodNames = getPodNamesForCrs(Set.of(kc)); + var prevRevision = getStatefulSet(kc).getStatus().getUpdateRevision(); var dbSecret = getDbSecret(); @@ -100,40 +104,29 @@ public class WatchedSecretsTest extends BaseOperatorTest { Base64.getEncoder().encodeToString(username.getBytes())); k8sclient.resource(dbSecret).update(); - Awaitility.await() - .ignoreExceptions() - .untilAsserted(() -> { - Log.info("Checking pod logs for DB auth failures"); - var podlogs = getPodNamesForCrs(Set.of(kc)).stream() - .filter(n -> !prevPodNames.contains(n)) // checking just new pods - .map(n -> { - var name = k8sclient - .pods() - .inNamespace(namespace) - .list() - .getItems() - .stream() - .filter(p -> (p.getMetadata().getName() + p.getMetadata().getCreationTimestamp()).equals(n)) - .findAny() - .get() - .getMetadata() - .getName(); + Pod pod = k8sclient.pods().withName(kc.getMetadata().getName() + "-0").waitUntilCondition( + p -> p != null && !prevRevision.equals(p.getMetadata().getLabels().get("controller-revision-hash")), + 30, TimeUnit.SECONDS); - return k8sclient.pods().inNamespace(namespace).withName(name).getLog(); - }) - .collect(Collectors.toList()); - assertThat(podlogs).anyMatch(l -> l.contains("password authentication failed for user \"" + username + "\"")); - }); + ByteArrayOutputStream logBytes = new ByteArrayOutputStream(); + try (var ignored = k8sclient.pods().resource(pod).watchLog(logBytes)) { + Awaitility.await().atMost(1, TimeUnit.MINUTES).until(() -> logBytes.toString(StandardCharsets.UTF_8) + .contains("password authentication failed for user \"" + username + "\"")); + } } catch (Exception e) { savePodLogs(); throw e; } } + private StatefulSet getStatefulSet(Keycloak kc) { + return k8sclient.apps().statefulSets().withName(kc.getMetadata().getName()).require(); + } + @Test public void testSecretsCanBeUnWatched() { try { - var kc = getDefaultKeycloakDeployment(); + var kc = getTestKeycloakDeployment(true); deployKeycloak(k8sclient, kc, true); Log.info("Updating KC to not to rely on DB Secret"); @@ -162,12 +155,12 @@ public class WatchedSecretsTest extends BaseOperatorTest { @Test public void testSingleSecretMultipleKeycloaks() { try { - var kc1 = getDefaultKeycloakDeployment(); + var kc1 = getTestKeycloakDeployment(true); var kc1Hostname = new HostnameSpecBuilder().withHostname("kc1.local").build(); kc1.getMetadata().setName(kc1.getMetadata().getName() + "-1"); kc1.getSpec().setHostnameSpec(kc1Hostname); - var kc2 = getDefaultKeycloakDeployment(); + var kc2 = getTestKeycloakDeployment(true); var kc2Hostname = new HostnameSpecBuilder().withHostname("kc2.local").build(); kc2.getMetadata().setName(kc2.getMetadata().getName() + "-2"); kc2.getSpec().setHostnameSpec(kc2Hostname); // to prevent Ingress conflicts @@ -204,12 +197,14 @@ public class WatchedSecretsTest extends BaseOperatorTest { private void testDeploymentRestarted(Set crsToBeRestarted, Set crsNotToBeRestarted, Runnable action) { boolean restartExpected = !crsToBeRestarted.isEmpty(); - List podsToBeRestarted = getPodNamesForCrs(crsToBeRestarted); - List podsNotToBeRestarted = getPodNamesForCrs(crsNotToBeRestarted); + var toBeRestarted = crsToBeRestarted.stream().collect(Collectors.toMap(Function.identity(), k -> getStatefulSet(k).getStatus().getUpdateRevision())); + var notToBeRestarted = crsNotToBeRestarted.stream().collect(Collectors.toMap(Function.identity(), k -> getStatefulSet(k).getStatus().getUpdateRevision())); action.run(); if (restartExpected) { + // this depends on the restart taking long enough to detect after the action is run + // we may want to switch to using an informer that runs before the action assertRollingUpdate(crsToBeRestarted, true); } @@ -220,45 +215,33 @@ public class WatchedSecretsTest extends BaseOperatorTest { if (restartExpected) { Awaitility.await() .untilAsserted(() -> { - List newPods = getPodNamesForCrs(allCrs); - Log.infof("Pods to be restarted: %s\nPods NOT to be restarted: %s\nCurrent Pods: %s", - podsToBeRestarted, podsNotToBeRestarted, newPods); - assertThat(newPods).noneMatch(podsToBeRestarted::contains); - assertThat(newPods).containsAll(podsNotToBeRestarted); + toBeRestarted.forEach((k, version) -> { + // make sure a new version was fully rolled in + var status = getStatefulSet(k).getStatus(); + assertThat(status.getUpdateRevision()).isEqualTo(status.getCurrentRevision()); + assertThat(status.getUpdateRevision()).isNotEqualTo(version); + }); }); } - else { + if (!notToBeRestarted.isEmpty()) { Awaitility.await() .during(10, TimeUnit.SECONDS) // to ensure no pods were created .untilAsserted(() -> { - List newPods = getPodNamesForCrs(allCrs); - Log.infof("Pods NOT to be restarted: %s, expected pods: %s\nAsserting current pods are unchanged: %s", - podsNotToBeRestarted, newPods); - assertThat(newPods).isEqualTo(podsNotToBeRestarted); + notToBeRestarted.forEach((k, version) -> { + // make sure the version has stayed the same + var status = getStatefulSet(k).getStatus(); + assertThat(status.getUpdateRevision()).isEqualTo(status.getCurrentRevision()); + assertThat(status.getUpdateRevision()).isEqualTo(version); + }); }); } } - private List getPodNamesForCrs(Set crs) { - return k8sclient - .pods() - .inNamespace(namespace) - .list() - .getItems() - .stream() - .map(pod -> pod.getMetadata().getName() + pod.getMetadata().getCreationTimestamp()) - .filter(pod -> crs.stream().map(c -> c.getMetadata().getName()).anyMatch(pod::startsWith)) - .collect(Collectors.toList()); - } - private void assertRollingUpdate(Set crs, boolean expectedStatus) { Awaitility.await() .untilAsserted(() -> { for (var cr : crs) { - Keycloak kc = k8sclient.resources(Keycloak.class) - .inNamespace(namespace) - .withName(cr.getMetadata().getName()) - .get(); + Keycloak kc = k8sclient.resource(cr).get(); assertKeycloakStatusCondition(kc, KeycloakStatusCondition.ROLLING_UPDATE, expectedStatus); } }); diff --git a/operator/src/test/java/org/keycloak/operator/testsuite/unit/PodTemplateTest.java b/operator/src/test/java/org/keycloak/operator/testsuite/unit/PodTemplateTest.java index bff4c00963..821dc47267 100644 --- a/operator/src/test/java/org/keycloak/operator/testsuite/unit/PodTemplateTest.java +++ b/operator/src/test/java/org/keycloak/operator/testsuite/unit/PodTemplateTest.java @@ -40,6 +40,7 @@ import org.keycloak.operator.crds.v2alpha1.deployment.spec.HttpSpecBuilder; import org.keycloak.operator.crds.v2alpha1.deployment.spec.UnsupportedSpec; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.function.Consumer; import java.util.function.Function; @@ -93,7 +94,7 @@ public class PodTemplateTest { var deployment = new KeycloakDeployment(null, config, kc, existingDeployment, "dummy-admin"); - return (StatefulSet) deployment.getReconciledResource().get(); + return deployment.getReconciledResource().get(); } private StatefulSet getDeployment(PodTemplateSpec podTemplate, StatefulSet existingDeployment) { @@ -368,4 +369,16 @@ public class PodTemplateTest { assertEquals("/some/health/ready", fourth.getReadinessProbe().getHttpGet().getPath()); assertEquals("/some/health/live", fourth.getLivenessProbe().getHttpGet().getPath()); } + + @Test + public void testDefaultArgs() { + // Arrange + PodTemplateSpec additionalPodTemplate = null; + + // Act + var podTemplate = getDeployment(additionalPodTemplate).getSpec().getTemplate(); + + // Assert + assertThat(podTemplate.getSpec().getContainers().get(0).getArgs()).doesNotContain("--optimized"); + } } diff --git a/operator/src/test/java/org/keycloak/operator/testsuite/utils/K8sUtils.java b/operator/src/test/java/org/keycloak/operator/testsuite/utils/K8sUtils.java index 7d9fa36db9..ac1282cdca 100644 --- a/operator/src/test/java/org/keycloak/operator/testsuite/utils/K8sUtils.java +++ b/operator/src/test/java/org/keycloak/operator/testsuite/utils/K8sUtils.java @@ -19,25 +19,29 @@ package org.keycloak.operator.testsuite.utils; import io.fabric8.kubernetes.api.model.HasMetadata; import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.PodBuilder; import io.fabric8.kubernetes.api.model.Secret; import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.KubernetesClientException; +import io.fabric8.kubernetes.client.dsl.ExecWatch; import io.fabric8.kubernetes.client.dsl.Resource; -import io.fabric8.kubernetes.client.utils.KubernetesResourceUtil; import io.fabric8.kubernetes.client.utils.Serialization; import io.quarkus.logging.Log; + import org.awaitility.Awaitility; import org.keycloak.operator.crds.v2alpha1.deployment.Keycloak; +import org.keycloak.operator.crds.v2alpha1.deployment.KeycloakSpecBuilder; import org.keycloak.operator.crds.v2alpha1.deployment.KeycloakStatusCondition; -import org.keycloak.operator.testsuite.integration.BaseOperatorTest; +import java.io.ByteArrayOutputStream; import java.io.InputStream; import java.net.HttpURLConnection; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Objects; -import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * @author Vaclav Muzikar @@ -48,16 +52,13 @@ public final class K8sUtils { } public static Keycloak getDefaultKeycloakDeployment() { - Keycloak kc = getResourceFromFile("example-keycloak.yaml", Keycloak.class); - kc.getMetadata().setNamespace(BaseOperatorTest.getCurrentNamespace()); - return kc; + return getResourceFromFile("example-keycloak.yaml", Keycloak.class); } public static Secret getDefaultTlsSecret() { return getResourceFromFile("example-tls-secret.yaml", Secret.class); } - public static void deployKeycloak(KubernetesClient client, Keycloak kc, boolean waitUntilReady) { deployKeycloak(client, kc, waitUntilReady, true); } @@ -83,6 +84,10 @@ public final class K8sUtils { set(client, getDefaultTlsSecret()); } + // speed the cleanup of pods + kc.setSpec(new KeycloakSpecBuilder(kc.getSpec()).editOrNewUnsupported().editOrNewPodTemplate().editOrNewSpec() + .withTerminationGracePeriodSeconds(0L).endSpec().endPodTemplate().endUnsupported().build()); + set(client, kc); if (waitUntilReady) { @@ -90,10 +95,6 @@ public final class K8sUtils { } } - public static void deployDefaultKeycloak(KubernetesClient client) { - deployKeycloak(client, getDefaultKeycloakDeployment(), true); - } - public static void waitForKeycloakToBeReady(KubernetesClient client, Keycloak kc) { Log.infof("Waiting for Keycloak \"%s\"", kc.getMetadata().getName()); Awaitility.await() @@ -113,37 +114,37 @@ public final class K8sUtils { } public static String inClusterCurl(KubernetesClient k8sclient, String namespace, String... args) { - var podName = KubernetesResourceUtil.sanitizeName("curl-" + UUID.randomUUID()); + var podName = "curl-pod"; try { - Pod curlPod = k8sclient.run().inNamespace(namespace) - .withNewRunConfig() - .withArgs(args) - .withName(podName) - .withImage("curlimages/curl:7.78.0") - .withRestartPolicy("Never") - .done(); - Log.info("Waiting for curl Pod to finish running"); - Awaitility.await().atMost(3, TimeUnit.MINUTES) - .until(() -> { - String phase = - k8sclient.pods().inNamespace(namespace).withName(podName).get() - .getStatus().getPhase(); - return phase.equals("Succeeded") || phase.equals("Failed"); - }); + Pod curlPod = new PodBuilder().withNewMetadata().withName(podName).endMetadata().withNewSpec() + .addNewContainer() + .withImage("curlimages/curl:8.1.2") + .withCommand("sh") + .withName("curl") + .withStdin() + .endContainer() + .endSpec() + .build(); - String curlOutput = - k8sclient.pods().inNamespace(namespace) - .withName(curlPod.getMetadata().getName()).getLog(); + try { + k8sclient.resource(curlPod).create(); + } catch (KubernetesClientException e) { + if (e.getCode() != HttpURLConnection.HTTP_CONFLICT) { + throw e; + } + } - return curlOutput; - } catch (KubernetesClientException ex) { - throw new AssertionError(ex); - } finally { - Log.info("Deleting curl Pod"); - k8sclient.pods().inNamespace(namespace).withName(podName).delete(); - Awaitility.await().atMost(2, TimeUnit.MINUTES) - .until(() -> k8sclient.pods().inNamespace(namespace).withName(podName) - .get() == null); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + + try (ExecWatch watch = k8sclient.pods().resource(curlPod).withReadyWaitTimeout(60000) + .writingOutput(output) + .exec(Stream.concat(Stream.of("curl"), Stream.of(args)).toArray(String[]::new))) { + watch.exitCode().get(15, TimeUnit.SECONDS); + } + + return output.toString(StandardCharsets.UTF_8); + } catch (Exception ex) { + throw KubernetesClientException.launderThrowable(ex); } } }