diff --git a/docs/guides/server/tracing.adoc b/docs/guides/server/tracing.adoc index ef181da7c8..d60b36ce7c 100644 --- a/docs/guides/server/tracing.adoc +++ b/docs/guides/server/tracing.adoc @@ -118,4 +118,18 @@ External callers can manipulate trace headers, parent spans can be injected, and Proper HTTP headers (especially `tracestate`) filtering and adequate measures of caller trust would need to be assessed. For more information, see the https://www.w3.org/TR/trace-context/#security-considerations[W3C Trace context] document. + +== Tracing in Kubernetes environment +When the tracing is enabled when using the {project_name} Operator, certain information about the deployment is propagated to the underlying containers. + +NOTE: There is no support for tracing configuration in {project_name} CR yet, so the `additionalOptions` can be used to the `tracing-enabled` property and other tracing options. + +You can filter out the required traces in your tracing backend based on their tags: + +* `service.name` - {project_name} deployment name +* `k8s.namespace.name` - Namespace +* `host.name` - Pod name + +{project_name} Operator automatically sets the `KC_TRACING_SERVICE_NAME` and `KC_TRACING_RESOURCE_ATTRIBUTES` environment variables for each {project_name} container included in pods it manages. + diff --git a/operator/src/main/java/org/keycloak/operator/controllers/KeycloakDeploymentDependentResource.java b/operator/src/main/java/org/keycloak/operator/controllers/KeycloakDeploymentDependentResource.java index fdef57e855..ec773ac2a2 100644 --- a/operator/src/main/java/org/keycloak/operator/controllers/KeycloakDeploymentDependentResource.java +++ b/operator/src/main/java/org/keycloak/operator/controllers/KeycloakDeploymentDependentResource.java @@ -77,7 +77,7 @@ public class KeycloakDeploymentDependentResource extends CRUDKubernetesDependent private static final List COPY_ENV = Arrays.asList("HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY"); private static final String ZONE_KEY = "topology.kubernetes.io/zone"; - + private static final String SERVICE_ACCOUNT_DIR = "/var/run/secrets/kubernetes.io/serviceaccount/"; private static final String SERVICE_CA_CRT = SERVICE_ACCOUNT_DIR + "service-ca.crt"; @@ -85,6 +85,10 @@ public class KeycloakDeploymentDependentResource extends CRUDKubernetesDependent public static final String KC_TRUSTSTORE_PATHS = "KC_TRUSTSTORE_PATHS"; + // Tracing + public static final String KC_TRACING_SERVICE_NAME = "KC_TRACING_SERVICE_NAME"; + public static final String KC_TRACING_RESOURCE_ATTRIBUTES = "KC_TRACING_RESOURCE_ATTRIBUTES"; + static final String JGROUPS_DNS_QUERY_PARAM = "-Djgroups.dns.query="; public static final String OPTIMIZED_ARG = "--optimized"; @@ -420,6 +424,21 @@ public class KeycloakDeploymentDependentResource extends CRUDKubernetesDependent // include the kube CA if the user is not controlling KC_TRUSTSTORE_PATHS via the unsupported or the additional varMap.putIfAbsent(KC_TRUSTSTORE_PATHS, new EnvVarBuilder().withName(KC_TRUSTSTORE_PATHS).withValue(truststores).build()); + varMap.putIfAbsent(KC_TRACING_SERVICE_NAME, + new EnvVarBuilder().withName(KC_TRACING_SERVICE_NAME) + .withValue(keycloakCR.getMetadata().getName()) + .build() + ); + + // Possible OTel k8s attributes convention can be found here: https://opentelemetry.io/docs/specs/semconv/attributes-registry/k8s/#kubernetes-attributes + var tracingAttributes = Map.of("k8s.namespace.name", keycloakCR.getMetadata().getNamespace()); + + varMap.putIfAbsent(KC_TRACING_RESOURCE_ATTRIBUTES, + new EnvVarBuilder().withName(KC_TRACING_RESOURCE_ATTRIBUTES) + .withValue(tracingAttributes.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(","))) + .build() + ); + var envVars = new ArrayList<>(varMap.values()); baseDeployment.getSpec().getTemplate().getSpec().getContainers().get(0).setEnv(envVars); diff --git a/operator/src/test/java/org/keycloak/operator/testsuite/integration/ClusteringTest.java b/operator/src/test/java/org/keycloak/operator/testsuite/integration/ClusteringTest.java index 65204a604a..a2cdb29cd7 100644 --- a/operator/src/test/java/org/keycloak/operator/testsuite/integration/ClusteringTest.java +++ b/operator/src/test/java/org/keycloak/operator/testsuite/integration/ClusteringTest.java @@ -17,7 +17,9 @@ package org.keycloak.operator.testsuite.integration; +import io.fabric8.kubernetes.api.model.EnvVar; import io.fabric8.kubernetes.api.model.HasMetadata; +import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.Secret; import io.fabric8.kubernetes.api.model.Service; import io.fabric8.kubernetes.api.model.apps.StatefulSet; @@ -41,12 +43,14 @@ import org.keycloak.operator.testsuite.utils.K8sUtils; import java.time.Duration; import java.util.Optional; +import java.util.function.Function; import com.fasterxml.jackson.databind.JsonNode; import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.SECONDS; import static org.assertj.core.api.Assertions.assertThat; +import static org.keycloak.operator.controllers.KeycloakDeploymentDependentResource.KC_TRACING_SERVICE_NAME; @QuarkusTest public class ClusteringTest extends BaseOperatorTest { @@ -88,6 +92,33 @@ public class ClusteringTest extends BaseOperatorTest { checkInstanceCount(1, Ingress.class, kc, kc1); checkInstanceCount(2, Service.class, kc, kc1); + // Tracing assertions + var pods = k8sclient + .pods() + .inNamespace(namespace) + .withLabels(Constants.DEFAULT_LABELS) + .list() + .getItems(); + + assertThat(pods.size()).isEqualTo(2); + + Function getTracingServiceName = (pod) -> pod.getSpec().getContainers().get(0).getEnv().stream() + .filter(f -> f.getName().equals(KC_TRACING_SERVICE_NAME)).findAny().map(EnvVar::getValue).orElse(null); + + var kc1Pod = pods.stream().filter(f -> f.getMetadata().getName().startsWith("another-example-")).findAny().orElse(null); + assertThat(kc1Pod).isNotNull(); + + var tracingServiceName1 = getTracingServiceName.apply(kc1Pod); + assertThat(tracingServiceName1).isNotNull(); + assertThat(tracingServiceName1).isEqualTo("another-example"); + + var kcPod = pods.stream().filter(f -> !f.equals(kc1Pod)).findAny().orElse(null); + assertThat(kcPod).isNotNull(); + + var tracingServiceName2 = getTracingServiceName.apply(kcPod); + assertThat(tracingServiceName2).isNotNull(); + assertThat(tracingServiceName2).isEqualTo("example-kc"); + // ensure they don't see each other's pods assertThat(k8sclient.resource(kc).scale().getStatus().getReplicas()).isEqualTo(1); assertThat(k8sclient.resource(kc1).scale().getStatus().getReplicas()).isEqualTo(1); diff --git a/operator/src/test/java/org/keycloak/operator/testsuite/integration/KeycloakDeploymentTest.java b/operator/src/test/java/org/keycloak/operator/testsuite/integration/KeycloakDeploymentTest.java index f7b2550bad..33ca3b0bcb 100644 --- a/operator/src/test/java/org/keycloak/operator/testsuite/integration/KeycloakDeploymentTest.java +++ b/operator/src/test/java/org/keycloak/operator/testsuite/integration/KeycloakDeploymentTest.java @@ -62,6 +62,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; import jakarta.inject.Inject; @@ -70,6 +71,8 @@ import static java.util.concurrent.TimeUnit.SECONDS; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.keycloak.operator.controllers.KeycloakDeploymentDependentResource.KC_TRACING_RESOURCE_ATTRIBUTES; +import static org.keycloak.operator.controllers.KeycloakDeploymentDependentResource.KC_TRACING_SERVICE_NAME; import static org.keycloak.operator.testsuite.utils.CRAssert.assertKeycloakStatusCondition; import static org.keycloak.operator.testsuite.utils.K8sUtils.deployKeycloak; import static org.keycloak.operator.testsuite.utils.K8sUtils.getResourceFromFile; @@ -383,14 +386,57 @@ public class KeycloakDeploymentTest extends BaseOperatorTest { assertKeycloakAccessibleViaService(kc, false, httpPort); } - + + @Test + public void testPodNamePropagation() { + var kc = getTestKeycloakDeployment(true); + kc.getSpec().getAdditionalOptions().add(new ValueOrSecret("tracing-enabled", "true")); + kc.getSpec().getAdditionalOptions().add(new ValueOrSecret("log-level", "io.opentelemetry:fine")); + deployKeycloak(k8sclient, kc, true); + + Awaitility.await() + .ignoreExceptions() + .untilAsserted(() -> + assertThat(k8sclient + .pods() + .inNamespace(namespace) + .withLabel("app", "keycloak") + .list() + .getItems() + .size()).isNotZero()); + + var pods = k8sclient + .pods() + .inNamespace(namespace) + .withLabels(Constants.DEFAULT_LABELS) + .list() + .getItems(); + + var envVars = pods.get(0).getSpec().getContainers().get(0).getEnv(); + assertThat(envVars).isNotNull(); + assertThat(envVars).isNotEmpty(); + + var serviceNameEnv = envVars.stream().filter(f -> f.getName().equals(KC_TRACING_SERVICE_NAME)).findAny().orElse(null); + assertThat(serviceNameEnv).isNotNull(); + assertThat(serviceNameEnv.getValue()).isEqualTo(kc.getMetadata().getName()); + + var resourceAttributesEnv = envVars.stream().filter(f -> f.getName().equals(KC_TRACING_RESOURCE_ATTRIBUTES)).findAny().orElse(null); + assertThat(resourceAttributesEnv).isNotNull(); + + var expectedAttributes = Map.of( + "k8s.namespace.name", kc.getMetadata().getNamespace() + ).entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(",")); + + assertThat(resourceAttributesEnv.getValue()).isEqualTo(expectedAttributes); + } + @Test public void testInitialAdminUser() { var kc = getTestKeycloakDeployment(true); String secretName = KeycloakAdminSecretDependentResource.getName(kc); assertInitialAdminUser(secretName, kc, false); } - + @Test public void testCustomBootstrapAdminUser() { var kc = getTestKeycloakDeployment(true); @@ -407,7 +453,7 @@ public class KeycloakDeploymentTest extends BaseOperatorTest { // Reference curl command: // curl --insecure --data "grant_type=password&client_id=admin-cli&username=admin&password=adminPassword" https://localhost:8443/realms/master/protocol/openid-connect/token public void assertInitialAdminUser(String secretName, Keycloak kc, boolean samePasswordAfterReinstall) { - + // Making sure no other Keycloak pod is still around Awaitility.await() .ignoreExceptions() diff --git a/operator/src/test/java/org/keycloak/operator/testsuite/unit/PodTemplateTest.java b/operator/src/test/java/org/keycloak/operator/testsuite/unit/PodTemplateTest.java index d7880bf729..f44f77b33b 100644 --- a/operator/src/test/java/org/keycloak/operator/testsuite/unit/PodTemplateTest.java +++ b/operator/src/test/java/org/keycloak/operator/testsuite/unit/PodTemplateTest.java @@ -73,22 +73,22 @@ public class PodTemplateTest { @InjectMock WatchedResources watchedResources; - + @Inject Config operatorConfig; @Inject KeycloakDistConfigurator distConfigurator; - + KeycloakDeploymentDependentResource deployment; - + @BeforeEach protected void setup() { this.deployment = new KeycloakDeploymentDependentResource(operatorConfig, watchedResources, distConfigurator); } private StatefulSet getDeployment(PodTemplateSpec podTemplate, StatefulSet existingDeployment, Consumer additionalSpec) { - var kc = new KeycloakBuilder().withNewMetadata().withName("instance").endMetadata().build(); + var kc = new KeycloakBuilder().withNewMetadata().withName("instance").withNamespace("keycloak-ns").endMetadata().build(); existingDeployment = new StatefulSetBuilder(existingDeployment).editOrNewSpec().editOrNewSelector() .withMatchLabels(Utils.allInstanceLabels(kc)) .endSelector().endSpec().build(); @@ -402,7 +402,11 @@ public class PodTemplateTest { // Assert assertNotNull(container); assertThat(container.getArgs()).doesNotContain(KeycloakDeploymentDependentResource.OPTIMIZED_ARG); - assertThat(container.getEnv().stream()).anyMatch(envVar -> envVar.getName().equals(KeycloakDeploymentDependentResource.KC_TRUSTSTORE_PATHS)); + + var envVars = container.getEnv(); + assertThat(envVars.stream()).anyMatch(envVar -> envVar.getName().equals(KeycloakDeploymentDependentResource.KC_TRUSTSTORE_PATHS)); + assertThat(envVars.stream()).anyMatch(envVar -> envVar.getName().equals(KeycloakDeploymentDependentResource.KC_TRACING_SERVICE_NAME)); + assertThat(envVars.stream()).anyMatch(envVar -> envVar.getName().equals(KeycloakDeploymentDependentResource.KC_TRACING_RESOURCE_ATTRIBUTES)); var readiness = container.getReadinessProbe().getHttpGet(); assertNotNull(readiness); @@ -421,29 +425,31 @@ public class PodTemplateTest { var affinity = podTemplate.getSpec().getAffinity(); assertNotNull(affinity); - assertThat(Serialization.asYaml(affinity)).isEqualTo("---\n" - + "podAffinity:\n" - + " preferredDuringSchedulingIgnoredDuringExecution:\n" - + " - podAffinityTerm:\n" - + " labelSelector:\n" - + " matchLabels:\n" - + " app: \"keycloak\"\n" - + " app.kubernetes.io/managed-by: \"keycloak-operator\"\n" - + " app.kubernetes.io/instance: \"instance\"\n" - + " app.kubernetes.io/component: \"server\"\n" - + " topologyKey: \"topology.kubernetes.io/zone\"\n" - + " weight: 10\n" - + "podAntiAffinity:\n" - + " preferredDuringSchedulingIgnoredDuringExecution:\n" - + " - podAffinityTerm:\n" - + " labelSelector:\n" - + " matchLabels:\n" - + " app: \"keycloak\"\n" - + " app.kubernetes.io/managed-by: \"keycloak-operator\"\n" - + " app.kubernetes.io/instance: \"instance\"\n" - + " app.kubernetes.io/component: \"server\"\n" - + " topologyKey: \"kubernetes.io/hostname\"\n" - + " weight: 50\n"); + assertThat(Serialization.asYaml(affinity)).isEqualTo(""" + --- + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app: "keycloak" + app.kubernetes.io/managed-by: "keycloak-operator" + app.kubernetes.io/instance: "instance" + app.kubernetes.io/component: "server" + topologyKey: "topology.kubernetes.io/zone" + weight: 10 + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app: "keycloak" + app.kubernetes.io/managed-by: "keycloak-operator" + app.kubernetes.io/instance: "instance" + app.kubernetes.io/component: "server" + topologyKey: "kubernetes.io/hostname" + weight: 50 + """); } @Test