OTEL: Dynamic service name for tracing in K8s environment (#32140)
* OTEL: Dynamic service name for tracing in K8s environment Closes #32095 Signed-off-by: Martin Bartoš <mabartos@redhat.com> * Update docs/guides/server/tracing.adoc Co-authored-by: Václav Muzikář <vaclav@muzikari.cz> Signed-off-by: Martin Bartoš <mabartos@redhat.com> --------- Signed-off-by: Martin Bartoš <mabartos@redhat.com> Co-authored-by: Václav Muzikář <vaclav@muzikari.cz>
This commit is contained in:
Martin Bartoš
GitHub
parent
087647dab3
commit
607ab01405
5 changed files with 148 additions and 32 deletions
|
|
@ -118,4 +118,18 @@ External callers can manipulate trace headers, parent spans can be injected, and
|
|||
Proper HTTP headers (especially `tracestate`) filtering and adequate measures of caller trust would need to be assessed.
|
||||
|
||||
For more information, see the https://www.w3.org/TR/trace-context/#security-considerations[W3C Trace context] document.
|
||||
|
||||
== Tracing in Kubernetes environment
|
||||
When the tracing is enabled when using the {project_name} Operator, certain information about the deployment is propagated to the underlying containers.
|
||||
|
||||
NOTE: There is no support for tracing configuration in {project_name} CR yet, so the `additionalOptions` can be used to the `tracing-enabled` property and other tracing options.
|
||||
|
||||
You can filter out the required traces in your tracing backend based on their tags:
|
||||
|
||||
* `service.name` - {project_name} deployment name
|
||||
* `k8s.namespace.name` - Namespace
|
||||
* `host.name` - Pod name
|
||||
|
||||
{project_name} Operator automatically sets the `KC_TRACING_SERVICE_NAME` and `KC_TRACING_RESOURCE_ATTRIBUTES` environment variables for each {project_name} container included in pods it manages.
|
||||
|
||||
</@tmpl.guide>
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ public class KeycloakDeploymentDependentResource extends CRUDKubernetesDependent
|
|||
private static final List<String> COPY_ENV = Arrays.asList("HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY");
|
||||
|
||||
private static final String ZONE_KEY = "topology.kubernetes.io/zone";
|
||||
|
||||
|
||||
private static final String SERVICE_ACCOUNT_DIR = "/var/run/secrets/kubernetes.io/serviceaccount/";
|
||||
private static final String SERVICE_CA_CRT = SERVICE_ACCOUNT_DIR + "service-ca.crt";
|
||||
|
||||
|
|
@ -85,6 +85,10 @@ public class KeycloakDeploymentDependentResource extends CRUDKubernetesDependent
|
|||
|
||||
public static final String KC_TRUSTSTORE_PATHS = "KC_TRUSTSTORE_PATHS";
|
||||
|
||||
// Tracing
|
||||
public static final String KC_TRACING_SERVICE_NAME = "KC_TRACING_SERVICE_NAME";
|
||||
public static final String KC_TRACING_RESOURCE_ATTRIBUTES = "KC_TRACING_RESOURCE_ATTRIBUTES";
|
||||
|
||||
static final String JGROUPS_DNS_QUERY_PARAM = "-Djgroups.dns.query=";
|
||||
|
||||
public static final String OPTIMIZED_ARG = "--optimized";
|
||||
|
|
@ -420,6 +424,21 @@ public class KeycloakDeploymentDependentResource extends CRUDKubernetesDependent
|
|||
// include the kube CA if the user is not controlling KC_TRUSTSTORE_PATHS via the unsupported or the additional
|
||||
varMap.putIfAbsent(KC_TRUSTSTORE_PATHS, new EnvVarBuilder().withName(KC_TRUSTSTORE_PATHS).withValue(truststores).build());
|
||||
|
||||
varMap.putIfAbsent(KC_TRACING_SERVICE_NAME,
|
||||
new EnvVarBuilder().withName(KC_TRACING_SERVICE_NAME)
|
||||
.withValue(keycloakCR.getMetadata().getName())
|
||||
.build()
|
||||
);
|
||||
|
||||
// Possible OTel k8s attributes convention can be found here: https://opentelemetry.io/docs/specs/semconv/attributes-registry/k8s/#kubernetes-attributes
|
||||
var tracingAttributes = Map.of("k8s.namespace.name", keycloakCR.getMetadata().getNamespace());
|
||||
|
||||
varMap.putIfAbsent(KC_TRACING_RESOURCE_ATTRIBUTES,
|
||||
new EnvVarBuilder().withName(KC_TRACING_RESOURCE_ATTRIBUTES)
|
||||
.withValue(tracingAttributes.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(",")))
|
||||
.build()
|
||||
);
|
||||
|
||||
var envVars = new ArrayList<>(varMap.values());
|
||||
baseDeployment.getSpec().getTemplate().getSpec().getContainers().get(0).setEnv(envVars);
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,9 @@
|
|||
|
||||
package org.keycloak.operator.testsuite.integration;
|
||||
|
||||
import io.fabric8.kubernetes.api.model.EnvVar;
|
||||
import io.fabric8.kubernetes.api.model.HasMetadata;
|
||||
import io.fabric8.kubernetes.api.model.Pod;
|
||||
import io.fabric8.kubernetes.api.model.Secret;
|
||||
import io.fabric8.kubernetes.api.model.Service;
|
||||
import io.fabric8.kubernetes.api.model.apps.StatefulSet;
|
||||
|
|
@ -41,12 +43,14 @@ import org.keycloak.operator.testsuite.utils.K8sUtils;
|
|||
|
||||
import java.time.Duration;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Function;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
|
||||
import static java.util.concurrent.TimeUnit.MINUTES;
|
||||
import static java.util.concurrent.TimeUnit.SECONDS;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.keycloak.operator.controllers.KeycloakDeploymentDependentResource.KC_TRACING_SERVICE_NAME;
|
||||
|
||||
@QuarkusTest
|
||||
public class ClusteringTest extends BaseOperatorTest {
|
||||
|
|
@ -88,6 +92,33 @@ public class ClusteringTest extends BaseOperatorTest {
|
|||
checkInstanceCount(1, Ingress.class, kc, kc1);
|
||||
checkInstanceCount(2, Service.class, kc, kc1);
|
||||
|
||||
// Tracing assertions
|
||||
var pods = k8sclient
|
||||
.pods()
|
||||
.inNamespace(namespace)
|
||||
.withLabels(Constants.DEFAULT_LABELS)
|
||||
.list()
|
||||
.getItems();
|
||||
|
||||
assertThat(pods.size()).isEqualTo(2);
|
||||
|
||||
Function<Pod, String> getTracingServiceName = (pod) -> pod.getSpec().getContainers().get(0).getEnv().stream()
|
||||
.filter(f -> f.getName().equals(KC_TRACING_SERVICE_NAME)).findAny().map(EnvVar::getValue).orElse(null);
|
||||
|
||||
var kc1Pod = pods.stream().filter(f -> f.getMetadata().getName().startsWith("another-example-")).findAny().orElse(null);
|
||||
assertThat(kc1Pod).isNotNull();
|
||||
|
||||
var tracingServiceName1 = getTracingServiceName.apply(kc1Pod);
|
||||
assertThat(tracingServiceName1).isNotNull();
|
||||
assertThat(tracingServiceName1).isEqualTo("another-example");
|
||||
|
||||
var kcPod = pods.stream().filter(f -> !f.equals(kc1Pod)).findAny().orElse(null);
|
||||
assertThat(kcPod).isNotNull();
|
||||
|
||||
var tracingServiceName2 = getTracingServiceName.apply(kcPod);
|
||||
assertThat(tracingServiceName2).isNotNull();
|
||||
assertThat(tracingServiceName2).isEqualTo("example-kc");
|
||||
|
||||
// ensure they don't see each other's pods
|
||||
assertThat(k8sclient.resource(kc).scale().getStatus().getReplicas()).isEqualTo(1);
|
||||
assertThat(k8sclient.resource(kc1).scale().getStatus().getReplicas()).isEqualTo(1);
|
||||
|
|
|
|||
|
|
@ -62,6 +62,7 @@ import java.util.concurrent.ExecutionException;
|
|||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import jakarta.inject.Inject;
|
||||
|
||||
|
|
@ -70,6 +71,8 @@ import static java.util.concurrent.TimeUnit.SECONDS;
|
|||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.keycloak.operator.controllers.KeycloakDeploymentDependentResource.KC_TRACING_RESOURCE_ATTRIBUTES;
|
||||
import static org.keycloak.operator.controllers.KeycloakDeploymentDependentResource.KC_TRACING_SERVICE_NAME;
|
||||
import static org.keycloak.operator.testsuite.utils.CRAssert.assertKeycloakStatusCondition;
|
||||
import static org.keycloak.operator.testsuite.utils.K8sUtils.deployKeycloak;
|
||||
import static org.keycloak.operator.testsuite.utils.K8sUtils.getResourceFromFile;
|
||||
|
|
@ -383,14 +386,57 @@ public class KeycloakDeploymentTest extends BaseOperatorTest {
|
|||
|
||||
assertKeycloakAccessibleViaService(kc, false, httpPort);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testPodNamePropagation() {
|
||||
var kc = getTestKeycloakDeployment(true);
|
||||
kc.getSpec().getAdditionalOptions().add(new ValueOrSecret("tracing-enabled", "true"));
|
||||
kc.getSpec().getAdditionalOptions().add(new ValueOrSecret("log-level", "io.opentelemetry:fine"));
|
||||
deployKeycloak(k8sclient, kc, true);
|
||||
|
||||
Awaitility.await()
|
||||
.ignoreExceptions()
|
||||
.untilAsserted(() ->
|
||||
assertThat(k8sclient
|
||||
.pods()
|
||||
.inNamespace(namespace)
|
||||
.withLabel("app", "keycloak")
|
||||
.list()
|
||||
.getItems()
|
||||
.size()).isNotZero());
|
||||
|
||||
var pods = k8sclient
|
||||
.pods()
|
||||
.inNamespace(namespace)
|
||||
.withLabels(Constants.DEFAULT_LABELS)
|
||||
.list()
|
||||
.getItems();
|
||||
|
||||
var envVars = pods.get(0).getSpec().getContainers().get(0).getEnv();
|
||||
assertThat(envVars).isNotNull();
|
||||
assertThat(envVars).isNotEmpty();
|
||||
|
||||
var serviceNameEnv = envVars.stream().filter(f -> f.getName().equals(KC_TRACING_SERVICE_NAME)).findAny().orElse(null);
|
||||
assertThat(serviceNameEnv).isNotNull();
|
||||
assertThat(serviceNameEnv.getValue()).isEqualTo(kc.getMetadata().getName());
|
||||
|
||||
var resourceAttributesEnv = envVars.stream().filter(f -> f.getName().equals(KC_TRACING_RESOURCE_ATTRIBUTES)).findAny().orElse(null);
|
||||
assertThat(resourceAttributesEnv).isNotNull();
|
||||
|
||||
var expectedAttributes = Map.of(
|
||||
"k8s.namespace.name", kc.getMetadata().getNamespace()
|
||||
).entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(","));
|
||||
|
||||
assertThat(resourceAttributesEnv.getValue()).isEqualTo(expectedAttributes);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInitialAdminUser() {
|
||||
var kc = getTestKeycloakDeployment(true);
|
||||
String secretName = KeycloakAdminSecretDependentResource.getName(kc);
|
||||
assertInitialAdminUser(secretName, kc, false);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testCustomBootstrapAdminUser() {
|
||||
var kc = getTestKeycloakDeployment(true);
|
||||
|
|
@ -407,7 +453,7 @@ public class KeycloakDeploymentTest extends BaseOperatorTest {
|
|||
// Reference curl command:
|
||||
// curl --insecure --data "grant_type=password&client_id=admin-cli&username=admin&password=adminPassword" https://localhost:8443/realms/master/protocol/openid-connect/token
|
||||
public void assertInitialAdminUser(String secretName, Keycloak kc, boolean samePasswordAfterReinstall) {
|
||||
|
||||
|
||||
// Making sure no other Keycloak pod is still around
|
||||
Awaitility.await()
|
||||
.ignoreExceptions()
|
||||
|
|
|
|||
|
|
@ -73,22 +73,22 @@ public class PodTemplateTest {
|
|||
|
||||
@InjectMock
|
||||
WatchedResources watchedResources;
|
||||
|
||||
|
||||
@Inject
|
||||
Config operatorConfig;
|
||||
|
||||
@Inject
|
||||
KeycloakDistConfigurator distConfigurator;
|
||||
|
||||
|
||||
KeycloakDeploymentDependentResource deployment;
|
||||
|
||||
|
||||
@BeforeEach
|
||||
protected void setup() {
|
||||
this.deployment = new KeycloakDeploymentDependentResource(operatorConfig, watchedResources, distConfigurator);
|
||||
}
|
||||
|
||||
private StatefulSet getDeployment(PodTemplateSpec podTemplate, StatefulSet existingDeployment, Consumer<KeycloakSpecBuilder> additionalSpec) {
|
||||
var kc = new KeycloakBuilder().withNewMetadata().withName("instance").endMetadata().build();
|
||||
var kc = new KeycloakBuilder().withNewMetadata().withName("instance").withNamespace("keycloak-ns").endMetadata().build();
|
||||
existingDeployment = new StatefulSetBuilder(existingDeployment).editOrNewSpec().editOrNewSelector()
|
||||
.withMatchLabels(Utils.allInstanceLabels(kc))
|
||||
.endSelector().endSpec().build();
|
||||
|
|
@ -402,7 +402,11 @@ public class PodTemplateTest {
|
|||
// Assert
|
||||
assertNotNull(container);
|
||||
assertThat(container.getArgs()).doesNotContain(KeycloakDeploymentDependentResource.OPTIMIZED_ARG);
|
||||
assertThat(container.getEnv().stream()).anyMatch(envVar -> envVar.getName().equals(KeycloakDeploymentDependentResource.KC_TRUSTSTORE_PATHS));
|
||||
|
||||
var envVars = container.getEnv();
|
||||
assertThat(envVars.stream()).anyMatch(envVar -> envVar.getName().equals(KeycloakDeploymentDependentResource.KC_TRUSTSTORE_PATHS));
|
||||
assertThat(envVars.stream()).anyMatch(envVar -> envVar.getName().equals(KeycloakDeploymentDependentResource.KC_TRACING_SERVICE_NAME));
|
||||
assertThat(envVars.stream()).anyMatch(envVar -> envVar.getName().equals(KeycloakDeploymentDependentResource.KC_TRACING_RESOURCE_ATTRIBUTES));
|
||||
|
||||
var readiness = container.getReadinessProbe().getHttpGet();
|
||||
assertNotNull(readiness);
|
||||
|
|
@ -421,29 +425,31 @@ public class PodTemplateTest {
|
|||
|
||||
var affinity = podTemplate.getSpec().getAffinity();
|
||||
assertNotNull(affinity);
|
||||
assertThat(Serialization.asYaml(affinity)).isEqualTo("---\n"
|
||||
+ "podAffinity:\n"
|
||||
+ " preferredDuringSchedulingIgnoredDuringExecution:\n"
|
||||
+ " - podAffinityTerm:\n"
|
||||
+ " labelSelector:\n"
|
||||
+ " matchLabels:\n"
|
||||
+ " app: \"keycloak\"\n"
|
||||
+ " app.kubernetes.io/managed-by: \"keycloak-operator\"\n"
|
||||
+ " app.kubernetes.io/instance: \"instance\"\n"
|
||||
+ " app.kubernetes.io/component: \"server\"\n"
|
||||
+ " topologyKey: \"topology.kubernetes.io/zone\"\n"
|
||||
+ " weight: 10\n"
|
||||
+ "podAntiAffinity:\n"
|
||||
+ " preferredDuringSchedulingIgnoredDuringExecution:\n"
|
||||
+ " - podAffinityTerm:\n"
|
||||
+ " labelSelector:\n"
|
||||
+ " matchLabels:\n"
|
||||
+ " app: \"keycloak\"\n"
|
||||
+ " app.kubernetes.io/managed-by: \"keycloak-operator\"\n"
|
||||
+ " app.kubernetes.io/instance: \"instance\"\n"
|
||||
+ " app.kubernetes.io/component: \"server\"\n"
|
||||
+ " topologyKey: \"kubernetes.io/hostname\"\n"
|
||||
+ " weight: 50\n");
|
||||
assertThat(Serialization.asYaml(affinity)).isEqualTo("""
|
||||
---
|
||||
podAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- podAffinityTerm:
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
app: "keycloak"
|
||||
app.kubernetes.io/managed-by: "keycloak-operator"
|
||||
app.kubernetes.io/instance: "instance"
|
||||
app.kubernetes.io/component: "server"
|
||||
topologyKey: "topology.kubernetes.io/zone"
|
||||
weight: 10
|
||||
podAntiAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- podAffinityTerm:
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
app: "keycloak"
|
||||
app.kubernetes.io/managed-by: "keycloak-operator"
|
||||
app.kubernetes.io/instance: "instance"
|
||||
app.kubernetes.io/component: "server"
|
||||
topologyKey: "kubernetes.io/hostname"
|
||||
weight: 50
|
||||
""");
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
|||
Loading…
Reference in a new issue