OTEL: Dynamic service name for tracing in K8s environment (#32140)

* OTEL: Dynamic service name for tracing in K8s environment

Closes #32095

Signed-off-by: Martin Bartoš <mabartos@redhat.com>

* Update docs/guides/server/tracing.adoc

Co-authored-by: Václav Muzikář <vaclav@muzikari.cz>
Signed-off-by: Martin Bartoš <mabartos@redhat.com>

---------

Signed-off-by: Martin Bartoš <mabartos@redhat.com>
Co-authored-by: Václav Muzikář <vaclav@muzikari.cz>

docs/guides/server/tracing.adoc

@@ -118,4 +118,18 @@ External callers can manipulate trace headers, parent spans can be injected, and
 Proper HTTP headers (especially `tracestate`) filtering and adequate measures of caller trust would need to be assessed.
 
 For more information, see the https://www.w3.org/TR/trace-context/#security-considerations[W3C Trace context] document.
+
+== Tracing in Kubernetes environment
+When the tracing is enabled when using the {project_name} Operator, certain information about the deployment is propagated to the underlying containers.
+
+NOTE: There is no support for tracing configuration in {project_name} CR yet, so the `additionalOptions` can be used to the `tracing-enabled` property and other tracing options.
+
+You can filter out the required traces in your tracing backend based on their tags:
+
+* `service.name` - {project_name} deployment name
+* `k8s.namespace.name` - Namespace
+* `host.name` - Pod name
+
+{project_name} Operator automatically sets the `KC_TRACING_SERVICE_NAME` and `KC_TRACING_RESOURCE_ATTRIBUTES` environment variables for each {project_name} container included in pods it manages.
+
 </@tmpl.guide>

operator/src/main/java/org/keycloak/operator/controllers/KeycloakDeploymentDependentResource.java

@@ -77,7 +77,7 @@ public class KeycloakDeploymentDependentResource extends CRUDKubernetesDependent
     private static final List<String> COPY_ENV = Arrays.asList("HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY");
 
     private static final String ZONE_KEY = "topology.kubernetes.io/zone";
-    
+
     private static final String SERVICE_ACCOUNT_DIR = "/var/run/secrets/kubernetes.io/serviceaccount/";
     private static final String SERVICE_CA_CRT = SERVICE_ACCOUNT_DIR + "service-ca.crt";
 
@@ -85,6 +85,10 @@ public class KeycloakDeploymentDependentResource extends CRUDKubernetesDependent
 
     public static final String KC_TRUSTSTORE_PATHS = "KC_TRUSTSTORE_PATHS";
 
+    // Tracing
+    public static final String KC_TRACING_SERVICE_NAME = "KC_TRACING_SERVICE_NAME";
+    public static final String KC_TRACING_RESOURCE_ATTRIBUTES = "KC_TRACING_RESOURCE_ATTRIBUTES";
+
     static final String JGROUPS_DNS_QUERY_PARAM = "-Djgroups.dns.query=";
 
     public static final String OPTIMIZED_ARG = "--optimized";
@@ -420,6 +424,21 @@ public class KeycloakDeploymentDependentResource extends CRUDKubernetesDependent
         // include the kube CA if the user is not controlling KC_TRUSTSTORE_PATHS via the unsupported or the additional
         varMap.putIfAbsent(KC_TRUSTSTORE_PATHS, new EnvVarBuilder().withName(KC_TRUSTSTORE_PATHS).withValue(truststores).build());
 
+        varMap.putIfAbsent(KC_TRACING_SERVICE_NAME,
+                new EnvVarBuilder().withName(KC_TRACING_SERVICE_NAME)
+                        .withValue(keycloakCR.getMetadata().getName())
+                        .build()
+        );
+
+        // Possible OTel k8s attributes convention can be found here: https://opentelemetry.io/docs/specs/semconv/attributes-registry/k8s/#kubernetes-attributes
+        var tracingAttributes = Map.of("k8s.namespace.name", keycloakCR.getMetadata().getNamespace());
+
+        varMap.putIfAbsent(KC_TRACING_RESOURCE_ATTRIBUTES,
+                new EnvVarBuilder().withName(KC_TRACING_RESOURCE_ATTRIBUTES)
+                        .withValue(tracingAttributes.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(",")))
+                        .build()
+        );
+
         var envVars = new ArrayList<>(varMap.values());
         baseDeployment.getSpec().getTemplate().getSpec().getContainers().get(0).setEnv(envVars);
 

operator/src/test/java/org/keycloak/operator/testsuite/integration/ClusteringTest.java

@@ -17,7 +17,9 @@
 
 package org.keycloak.operator.testsuite.integration;
 
+import io.fabric8.kubernetes.api.model.EnvVar;
 import io.fabric8.kubernetes.api.model.HasMetadata;
+import io.fabric8.kubernetes.api.model.Pod;
 import io.fabric8.kubernetes.api.model.Secret;
 import io.fabric8.kubernetes.api.model.Service;
 import io.fabric8.kubernetes.api.model.apps.StatefulSet;
@@ -41,12 +43,14 @@ import org.keycloak.operator.testsuite.utils.K8sUtils;
 
 import java.time.Duration;
 import java.util.Optional;
+import java.util.function.Function;
 
 import com.fasterxml.jackson.databind.JsonNode;
 
 import static java.util.concurrent.TimeUnit.MINUTES;
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.keycloak.operator.controllers.KeycloakDeploymentDependentResource.KC_TRACING_SERVICE_NAME;
 
 @QuarkusTest
 public class ClusteringTest extends BaseOperatorTest {
@@ -88,6 +92,33 @@ public class ClusteringTest extends BaseOperatorTest {
         checkInstanceCount(1, Ingress.class, kc, kc1);
         checkInstanceCount(2, Service.class, kc, kc1);
 
+        // Tracing assertions
+        var pods = k8sclient
+                .pods()
+                .inNamespace(namespace)
+                .withLabels(Constants.DEFAULT_LABELS)
+                .list()
+                .getItems();
+
+        assertThat(pods.size()).isEqualTo(2);
+
+        Function<Pod, String> getTracingServiceName = (pod) -> pod.getSpec().getContainers().get(0).getEnv().stream()
+                .filter(f -> f.getName().equals(KC_TRACING_SERVICE_NAME)).findAny().map(EnvVar::getValue).orElse(null);
+
+        var kc1Pod = pods.stream().filter(f -> f.getMetadata().getName().startsWith("another-example-")).findAny().orElse(null);
+        assertThat(kc1Pod).isNotNull();
+
+        var tracingServiceName1 = getTracingServiceName.apply(kc1Pod);
+        assertThat(tracingServiceName1).isNotNull();
+        assertThat(tracingServiceName1).isEqualTo("another-example");
+
+        var kcPod = pods.stream().filter(f -> !f.equals(kc1Pod)).findAny().orElse(null);
+        assertThat(kcPod).isNotNull();
+
+        var tracingServiceName2 = getTracingServiceName.apply(kcPod);
+        assertThat(tracingServiceName2).isNotNull();
+        assertThat(tracingServiceName2).isEqualTo("example-kc");
+
         // ensure they don't see each other's pods
         assertThat(k8sclient.resource(kc).scale().getStatus().getReplicas()).isEqualTo(1);
         assertThat(k8sclient.resource(kc1).scale().getStatus().getReplicas()).isEqualTo(1);

operator/src/test/java/org/keycloak/operator/testsuite/integration/KeycloakDeploymentTest.java

@@ -62,6 +62,7 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
 
 import jakarta.inject.Inject;
 
@@ -70,6 +71,8 @@ import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.keycloak.operator.controllers.KeycloakDeploymentDependentResource.KC_TRACING_RESOURCE_ATTRIBUTES;
+import static org.keycloak.operator.controllers.KeycloakDeploymentDependentResource.KC_TRACING_SERVICE_NAME;
 import static org.keycloak.operator.testsuite.utils.CRAssert.assertKeycloakStatusCondition;
 import static org.keycloak.operator.testsuite.utils.K8sUtils.deployKeycloak;
 import static org.keycloak.operator.testsuite.utils.K8sUtils.getResourceFromFile;
@@ -383,14 +386,57 @@ public class KeycloakDeploymentTest extends BaseOperatorTest {
 
         assertKeycloakAccessibleViaService(kc, false, httpPort);
     }
-    
+
+    @Test
+    public void testPodNamePropagation() {
+        var kc = getTestKeycloakDeployment(true);
+        kc.getSpec().getAdditionalOptions().add(new ValueOrSecret("tracing-enabled", "true"));
+        kc.getSpec().getAdditionalOptions().add(new ValueOrSecret("log-level", "io.opentelemetry:fine"));
+        deployKeycloak(k8sclient, kc, true);
+
+        Awaitility.await()
+                .ignoreExceptions()
+                .untilAsserted(() ->
+                        assertThat(k8sclient
+                                .pods()
+                                .inNamespace(namespace)
+                                .withLabel("app", "keycloak")
+                                .list()
+                                .getItems()
+                                .size()).isNotZero());
+
+        var pods = k8sclient
+                .pods()
+                .inNamespace(namespace)
+                .withLabels(Constants.DEFAULT_LABELS)
+                .list()
+                .getItems();
+
+        var envVars = pods.get(0).getSpec().getContainers().get(0).getEnv();
+        assertThat(envVars).isNotNull();
+        assertThat(envVars).isNotEmpty();
+
+        var serviceNameEnv = envVars.stream().filter(f -> f.getName().equals(KC_TRACING_SERVICE_NAME)).findAny().orElse(null);
+        assertThat(serviceNameEnv).isNotNull();
+        assertThat(serviceNameEnv.getValue()).isEqualTo(kc.getMetadata().getName());
+
+        var resourceAttributesEnv = envVars.stream().filter(f -> f.getName().equals(KC_TRACING_RESOURCE_ATTRIBUTES)).findAny().orElse(null);
+        assertThat(resourceAttributesEnv).isNotNull();
+
+        var expectedAttributes = Map.of(
+                "k8s.namespace.name", kc.getMetadata().getNamespace()
+        ).entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(","));
+
+        assertThat(resourceAttributesEnv.getValue()).isEqualTo(expectedAttributes);
+    }
+
     @Test
     public void testInitialAdminUser() {
         var kc = getTestKeycloakDeployment(true);
         String secretName = KeycloakAdminSecretDependentResource.getName(kc);
         assertInitialAdminUser(secretName, kc, false);
     }
-    
+
     @Test
     public void testCustomBootstrapAdminUser() {
         var kc = getTestKeycloakDeployment(true);
@@ -407,7 +453,7 @@ public class KeycloakDeploymentTest extends BaseOperatorTest {
     // Reference curl command:
     // curl --insecure --data "grant_type=password&client_id=admin-cli&username=admin&password=adminPassword" https://localhost:8443/realms/master/protocol/openid-connect/token
     public void assertInitialAdminUser(String secretName, Keycloak kc, boolean samePasswordAfterReinstall) {
-        
+
         // Making sure no other Keycloak pod is still around
         Awaitility.await()
                 .ignoreExceptions()

operator/src/test/java/org/keycloak/operator/testsuite/unit/PodTemplateTest.java

@@ -73,22 +73,22 @@ public class PodTemplateTest {
 
     @InjectMock
     WatchedResources watchedResources;
-    
+
     @Inject
     Config operatorConfig;
 
     @Inject
     KeycloakDistConfigurator distConfigurator;
-    
+
     KeycloakDeploymentDependentResource deployment;
-    
+
     @BeforeEach
     protected void setup() {
         this.deployment = new KeycloakDeploymentDependentResource(operatorConfig, watchedResources, distConfigurator);
     }
 
     private StatefulSet getDeployment(PodTemplateSpec podTemplate, StatefulSet existingDeployment, Consumer<KeycloakSpecBuilder> additionalSpec) {
-        var kc = new KeycloakBuilder().withNewMetadata().withName("instance").endMetadata().build();
+        var kc = new KeycloakBuilder().withNewMetadata().withName("instance").withNamespace("keycloak-ns").endMetadata().build();
         existingDeployment = new StatefulSetBuilder(existingDeployment).editOrNewSpec().editOrNewSelector()
                 .withMatchLabels(Utils.allInstanceLabels(kc))
                 .endSelector().endSpec().build();
@@ -402,7 +402,11 @@ public class PodTemplateTest {
         // Assert
         assertNotNull(container);
         assertThat(container.getArgs()).doesNotContain(KeycloakDeploymentDependentResource.OPTIMIZED_ARG);
-        assertThat(container.getEnv().stream()).anyMatch(envVar -> envVar.getName().equals(KeycloakDeploymentDependentResource.KC_TRUSTSTORE_PATHS));
+
+        var envVars = container.getEnv();
+        assertThat(envVars.stream()).anyMatch(envVar -> envVar.getName().equals(KeycloakDeploymentDependentResource.KC_TRUSTSTORE_PATHS));
+        assertThat(envVars.stream()).anyMatch(envVar -> envVar.getName().equals(KeycloakDeploymentDependentResource.KC_TRACING_SERVICE_NAME));
+        assertThat(envVars.stream()).anyMatch(envVar -> envVar.getName().equals(KeycloakDeploymentDependentResource.KC_TRACING_RESOURCE_ATTRIBUTES));
 
         var readiness = container.getReadinessProbe().getHttpGet();
         assertNotNull(readiness);
@@ -421,29 +425,31 @@ public class PodTemplateTest {
 
         var affinity = podTemplate.getSpec().getAffinity();
         assertNotNull(affinity);
-        assertThat(Serialization.asYaml(affinity)).isEqualTo("---\n"
+        assertThat(Serialization.asYaml(affinity)).isEqualTo("""
-                + "podAffinity:\n"
+                ---
-                + "  preferredDuringSchedulingIgnoredDuringExecution:\n"
+                podAffinity:
-                + "  - podAffinityTerm:\n"
+                  preferredDuringSchedulingIgnoredDuringExecution:
-                + "      labelSelector:\n"
+                  - podAffinityTerm:
-                + "        matchLabels:\n"
+                      labelSelector:
-                + "          app: \"keycloak\"\n"
+                        matchLabels:
-                + "          app.kubernetes.io/managed-by: \"keycloak-operator\"\n"
+                          app: "keycloak"
-                + "          app.kubernetes.io/instance: \"instance\"\n"
+                          app.kubernetes.io/managed-by: "keycloak-operator"
-                + "          app.kubernetes.io/component: \"server\"\n"
+                          app.kubernetes.io/instance: "instance"
-                + "      topologyKey: \"topology.kubernetes.io/zone\"\n"
+                          app.kubernetes.io/component: "server"
-                + "    weight: 10\n"
+                      topologyKey: "topology.kubernetes.io/zone"
-                + "podAntiAffinity:\n"
+                    weight: 10
-                + "  preferredDuringSchedulingIgnoredDuringExecution:\n"
+                podAntiAffinity:
-                + "  - podAffinityTerm:\n"
+                  preferredDuringSchedulingIgnoredDuringExecution:
-                + "      labelSelector:\n"
+                  - podAffinityTerm:
-                + "        matchLabels:\n"
+                      labelSelector:
-                + "          app: \"keycloak\"\n"
+                        matchLabels:
-                + "          app.kubernetes.io/managed-by: \"keycloak-operator\"\n"
+                          app: "keycloak"
-                + "          app.kubernetes.io/instance: \"instance\"\n"
+                          app.kubernetes.io/managed-by: "keycloak-operator"
-                + "          app.kubernetes.io/component: \"server\"\n"
+                          app.kubernetes.io/instance: "instance"
-                + "      topologyKey: \"kubernetes.io/hostname\"\n"
+                          app.kubernetes.io/component: "server"
-                + "    weight: 50\n");
+                      topologyKey: "kubernetes.io/hostname"
+                    weight: 50
+                """);
     }
 
     @Test