Provide histograms for http server metrics

Closes #28178

Co-authored-by: Martin Bartoš <mabartos@redhat.com>
Signed-off-by: Martin Bartoš <mabartos@redhat.com>
Signed-off-by: Alexander Schwartz <aschwart@redhat.com>

docs/documentation/upgrading/topics/changes/changes-25_0_0.adoc

@@ -15,6 +15,8 @@ http_server_requests_seconds_count{method="GET",outcome="SUCCESS",status="200",u
 http_server_requests_seconds_sum{method="GET",outcome="SUCCESS",status="200",uri="/realms/{realm}/protocol/{protocol}/auth"} 0.048717142
 ----
 
+Use the new options `http-metrics-histograms-enabled` and `http-metrics-slos` to enable default histogram buckets or specific buckets for service level objectives (SLOs).
+Read more about histograms in the https://prometheus.io/docs/concepts/metric_types/#histogram[Prometheus documentation about histograms] on how to use the additional metrics series provided in `http_server_requests_seconds_bucket`.
 
 = Nonce claim is only added to the ID token
 

docs/guides/server/configuration-metrics.adoc

@@ -6,7 +6,7 @@
 <@tmpl.guide
 title="Enabling {project_name} Metrics"
 summary="Learn how to enable and expose metrics from the server"
-includedOptions="metrics-enabled">
+includedOptions="metrics-enabled http-metrics-* cache-metrics-*">
 
 {project_name} has built in support for metrics. This {section} describes how to enable and configure server metrics.
 

quarkus/config-api/src/main/java/org/keycloak/config/HttpOptions.java

@@ -132,4 +132,16 @@ public class HttpOptions {
                          "If there are 48 processors it will be 192.")
             .build();
 
+    public static final Option<Boolean> HTTP_METRICS_HISTOGRAMS_ENABLED = new OptionBuilder<>("http-metrics-histograms-enabled", Boolean.class)
+            .category(OptionCategory.HTTP)
+            .description("Enables a histogram with default buckets for the duration of HTTP server requests.")
+            .defaultValue(Boolean.FALSE)
+            .build();
+
+    public static final Option<String> HTTP_METRICS_SLOS = new OptionBuilder<>("http-metrics-slos", String.class)
+            .category(OptionCategory.HTTP)
+            .description("Service level objectives for HTTP server requests. Use this instead of the default histogram, or use it in combination to add additional buckets. " +
+                    "Specify a list of comma-separated values defined in milliseconds. Example with buckets from 5ms to 10s: 5,10,25,50,250,500,1000,2500,5000,10000")
+            .build();
+
 }

quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/configuration/mappers/HttpPropertyMappers.java

@@ -114,6 +114,13 @@ public final class HttpPropertyMappers {
                         .to("quarkus.thread-pool.max-threads")
                         .transformer(HttpPropertyMappers::resolveMaxThreads)
                         .paramLabel("threads")
+                        .build(),
+                fromOption(HttpOptions.HTTP_METRICS_HISTOGRAMS_ENABLED)
+                        .isEnabled(MetricsPropertyMappers::metricsEnabled, MetricsPropertyMappers.METRICS_ENABLED_MSG)
+                        .build(),
+                fromOption(HttpOptions.HTTP_METRICS_SLOS)
+                        .isEnabled(MetricsPropertyMappers::metricsEnabled, MetricsPropertyMappers.METRICS_ENABLED_MSG)
+                        .paramLabel("list of buckets")
                         .build()
         };
     }

quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/services/metrics/HistogramMeterFilter.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright 2024 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.quarkus.runtime.services.metrics;
+
+import io.micrometer.core.instrument.Meter;
+import io.micrometer.core.instrument.config.MeterFilter;
+import io.micrometer.core.instrument.distribution.DistributionStatisticConfig;
+import jakarta.inject.Singleton;
+import org.keycloak.config.HttpOptions;
+import org.keycloak.quarkus.runtime.configuration.Configuration;
+
+import java.util.Arrays;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author Alexander Schwartz
+ */
+@Singleton
+public class HistogramMeterFilter implements MeterFilter {
+
+    private boolean histogramsEnabled;
+    private double[] slos;
+
+    public HistogramMeterFilter() {
+        histogramsEnabled = Configuration.isTrue(HttpOptions.HTTP_METRICS_HISTOGRAMS_ENABLED);
+        Optional<String> slosOption = Configuration.getOptionalKcValue(HttpOptions.HTTP_METRICS_SLOS.getKey());
+        if (slosOption.isPresent()) {
+            slos = Arrays.stream(slosOption.get().split(",")).filter(s -> !s.trim().isEmpty()).mapToDouble(s -> TimeUnit.MILLISECONDS.toNanos(Long.parseLong(s))).toArray();
+            if (slos.length == 0) {
+                slos = null;
+            }
+        }
+    }
+
+    @Override
+    public DistributionStatisticConfig configure(Meter.Id id, DistributionStatisticConfig config) {
+        if (isHttpServerRequests(id)) {
+            DistributionStatisticConfig.Builder builder = DistributionStatisticConfig.builder()
+                    .percentilesHistogram(histogramsEnabled);
+            if (slos != null) {
+                builder.serviceLevelObjectives(slos);
+            }
+            return builder.build().merge(config);
+        }
+        return config;
+    }
+
+    private boolean isHttpServerRequests(Meter.Id id) {
+        return "http.server.requests".equals(id.getName());
+    }
+}

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/MetricsDistTest.java

@@ -59,11 +59,18 @@ public class MetricsDistTest {
     }
 
     @Test
-    @Launch({ "start-dev", "--metrics-enabled=true", "--cache-metrics-histograms-enabled=true" })
+    @Launch({ "start-dev", "--metrics-enabled=true", "--cache-metrics-histograms-enabled=true", "--http-metrics-slos=5,10,25,50,250,500", "--http-metrics-histograms-enabled=true" })
     void testMetricsEndpointWithCacheMetricsHistograms() {
         when().get("/metrics").then()
                 .statusCode(200)
                 .body(containsString("vendor_statistics_miss_times_seconds_bucket"));
+
+        // histograms are only available at the second request as they then contain the metrics of the first request
+        when().get("/metrics").then()
+                .statusCode(200)
+                .body(containsString("http_server_requests_seconds_bucket{method=\"GET\",outcome=\"SUCCESS\",status=\"200\",uri=\"/metrics\",le=\"0.005\"}"))
+                .body(containsString("http_server_requests_seconds_bucket{method=\"GET\",outcome=\"SUCCESS\",status=\"200\",uri=\"/metrics\",le=\"0.005592405\"}"));
+
     }
 
     @Test

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testStartDevHelpAll.approved.txt

@@ -199,6 +199,15 @@ HTTP(S):
 --http-max-queued-requests <requests>
                      Maximum number of queued HTTP requests. Use this to shed load in an overload
                        situation. Excess requests will return a "503 Server not Available" response.
+--http-metrics-histograms-enabled <true|false>
+                     Enables a histogram with default buckets for the duration of HTTP server
+                       requests. Default: false. Available only when metrics are enabled.
+--http-metrics-slos <list of buckets>
+                     Service level objectives for HTTP server requests. Use this instead of the
+                       default histogram, or use it in combination to add additional buckets.
+                       Specify a list of comma-separated values defined in milliseconds. Example
+                       with buckets from 5ms to 10s: 5,10,25,50,250,500,1000,2500,5000,10000
+                       Available only when metrics are enabled.
 --http-pool-max-threads <threads>
                      The maximum number of threads. If this is not specified then it will be
                        automatically sized to the greater of 4 * the number of available processors

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testStartHelpAll.approved.txt

@@ -200,6 +200,15 @@ HTTP(S):
 --http-max-queued-requests <requests>
                      Maximum number of queued HTTP requests. Use this to shed load in an overload
                        situation. Excess requests will return a "503 Server not Available" response.
+--http-metrics-histograms-enabled <true|false>
+                     Enables a histogram with default buckets for the duration of HTTP server
+                       requests. Default: false. Available only when metrics are enabled.
+--http-metrics-slos <list of buckets>
+                     Service level objectives for HTTP server requests. Use this instead of the
+                       default histogram, or use it in combination to add additional buckets.
+                       Specify a list of comma-separated values defined in milliseconds. Example
+                       with buckets from 5ms to 10s: 5,10,25,50,250,500,1000,2500,5000,10000
+                       Available only when metrics are enabled.
 --http-pool-max-threads <threads>
                      The maximum number of threads. If this is not specified then it will be
                        automatically sized to the greater of 4 * the number of available processors

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testStartOptimizedHelpAll.approved.txt

@@ -173,6 +173,15 @@ HTTP(S):
 --http-max-queued-requests <requests>
                      Maximum number of queued HTTP requests. Use this to shed load in an overload
                        situation. Excess requests will return a "503 Server not Available" response.
+--http-metrics-histograms-enabled <true|false>
+                     Enables a histogram with default buckets for the duration of HTTP server
+                       requests. Default: false. Available only when metrics are enabled.
+--http-metrics-slos <list of buckets>
+                     Service level objectives for HTTP server requests. Use this instead of the
+                       default histogram, or use it in combination to add additional buckets.
+                       Specify a list of comma-separated values defined in milliseconds. Example
+                       with buckets from 5ms to 10s: 5,10,25,50,250,500,1000,2500,5000,10000
+                       Available only when metrics are enabled.
 --http-pool-max-threads <threads>
                      The maximum number of threads. If this is not specified then it will be
                        automatically sized to the greater of 4 * the number of available processors