Provide missing user event metrics from aerogear/keycloak-metrics-spi to a keycloak micrometer event listener

inspired by
https://github.com/aerogear/keycloak-metrics-spi
https://github.com/please-openit/keycloak-native-metrics

Closes #33043

Signed-off-by: Bernd Bohmann <bommel@apache.org>
Signed-off-by: Alexander Schwartz <aschwart@redhat.com>
Signed-off-by: Michal Hajas <mhajas@redhat.com>
Co-authored-by: Alexander Schwartz <aschwart@redhat.com>
Co-authored-by: Michal Hajas <mhajas@redhat.com>

common/src/main/java/org/keycloak/common/Profile.java

@@ -121,6 +121,8 @@ public class Profile {
         PASSKEYS("Passkeys", Type.PREVIEW),
 
         CACHE_EMBEDDED_REMOTE_STORE("Support for remote-store in embedded Infinispan caches", Type.EXPERIMENTAL),
+
+        USER_EVENT_METRICS("Collect metrics based on user events", Type.PREVIEW),
         ;
 
         private final Type type;

docs/guides/server/configuration-metrics.adoc

@@ -76,6 +76,9 @@ The table below summarizes the available metrics groups:
 |Cache
 |A set of metrics from Infinispan caches. See <@links.server id="caching"/> for more details.
 
+|Keycloak
+|A set of metrics from Keycloak events. See <@links.server id="event-metrics"/> for more details.
+
 |===
 
 </@tmpl.guide>

docs/guides/server/event-metrics.adoc

@@ -0,0 +1,60 @@
+<#import "/templates/guide.adoc" as tmpl>
+<#import "/templates/kc.adoc" as kc>
+<#import "/templates/options.adoc" as opts>
+<#import "/templates/links.adoc" as links>
+
+<@tmpl.guide
+title="Enabling {project_name} Event Metrics"
+summary="Learn how to enable and use {project_name} Event Metrics"
+preview="true"
+includedOptions="metrics-enabled event-metrics-user-*">
+
+Event metrics can provide admins an overview of the different activities in a {project_name} instance.
+For now, only metrics for user events are captured.
+For example, you can monitor the number of logins, login failures, or token refreshes performed.
+
+The metrics are exposed using the standard metrics endpoint, and you can use it in your own metrics collection system to create dashboards and alerts.
+
+The metrics are reported as counters per {project_name} instance.
+The counters are reset on the restart of the instance.
+If you have multiple instances running in a cluster, you will need to collect the metrics from all instances and aggregate them to get per a cluster view.
+
+== Enable event metrics
+
+To start collecting metrics, enable the feature `user-event-metrics`, enable metrics, and enable the metrics for user events.
+
+The following shows the required startup parameters:
+
+<@kc.start parameters="--features=user-event-metrics --metrics-enabled=true --event-metrics-user-enabled=true ..."/>
+
+By default, there is a separate metric for each realm.
+To break down the metric by client and identity provider, you can add those metrics dimension using the configuration option `event-metrics-user-tags`.
+This can be useful on installations with a small number of clients and IDPs.
+This is not recommended for installations with a large number of clients or IDPs as it will increase the memory usage of {project_name} and as it will increase the load on your monitoring system.
+
+The following shows how to configure {project_name} to break down the metrics by all three metrics dimensions:
+
+<@kc.start parameters="... --event-metrics-user-tags=realm,idp,clientId ..."/>
+
+You can limit the events for which {project_name} will expose metrics.
+
+The following example limits the events collected to `LOGIN` and `LOGOUT` events:
+
+<@kc.start parameters="... --event-metrics-user-events=login,logout ..."/>
+
+All error events will be collected with the primary event type and will have the `error` tag filled with the error code.
+
+The snippet below is an example of a response provided by the metric endpoint:
+
+[source]
+----
+# HELP keycloak_user_events_total Keycloak user events
+# TYPE keycloak_user_events_total counter
+keycloak_user_events_total{client_id="security-admin-console",error="",event="code_to_token",idp="",realm="master",} 1.0
+keycloak_user_events_total{client_id="security-admin-console",error="",event="login",idp="",realm="master",} 1.0
+keycloak_user_events_total{client_id="security-admin-console",error="",event="logout",idp="",realm="master",} 1.0
+keycloak_user_events_total{client_id="security-admin-console",error="invalid_user_credentials",event="login",idp="",realm="master",} 1.0
+----
+
+
+</@tmpl.guide>

docs/guides/server/pinned-guides

@@ -18,6 +18,7 @@ fips
 management-interface
 health
 configuration-metrics
+event-metrics
 tracing
 importExport
 vault

quarkus/config-api/src/main/java/org/keycloak/config/EventOptions.java

@@ -0,0 +1,30 @@
+package org.keycloak.config;
+
+import java.util.List;
+
+public class EventOptions {
+
+    public static final Option<Boolean> USER_EVENT_METRICS_ENABLED = new OptionBuilder<>("event-metrics-user-enabled", Boolean.class)
+            .category(OptionCategory.EVENTS)
+            .description("Create metrics based on user events.")
+            .buildTime(true)
+            .defaultValue(Boolean.FALSE)
+            .build();
+
+    public static final Option<List<String>> USER_EVENT_METRICS_TAGS = OptionBuilder.listOptionBuilder("event-metrics-user-tags", String.class)
+            .category(OptionCategory.EVENTS)
+            .description("Comma-separated list of tags to be collected for user event metrics. By default only 'realm' is enabled to avoid a high metrics cardinality.")
+            .buildTime(false)
+            .expectedValues(List.of("realm", "idp", "clientId"))
+            .defaultValue(List.of("realm"))
+            .build();
+
+    public static final Option<List<String>> USER_EVENT_METRICS_EVENTS = OptionBuilder.listOptionBuilder("event-metrics-user-events", String.class)
+            .category(OptionCategory.EVENTS)
+            .description("Comma-separated list of events to be collected for user event metrics. Reduce the number of metrics. If empty or not set, all events create a metric.")
+            .buildTime(false)
+            .build();
+
+}
+
+

quarkus/config-api/src/main/java/org/keycloak/config/OptionCategory.java

@@ -16,6 +16,7 @@ public enum OptionCategory {
     VAULT("Vault", 100, ConfigSupportLevel.SUPPORTED),
     LOGGING("Logging", 110, ConfigSupportLevel.SUPPORTED),
     TRACING("Tracing", 111, ConfigSupportLevel.PREVIEW),
+    EVENTS("Events", 112, ConfigSupportLevel.PREVIEW),
     TRUSTSTORE("Truststore", 115, ConfigSupportLevel.SUPPORTED),
     SECURITY("Security", 120, ConfigSupportLevel.SUPPORTED),
     EXPORT("Export", 130, ConfigSupportLevel.SUPPORTED),

quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/configuration/mappers/EventPropertyMappers.java

@@ -0,0 +1,45 @@
+package org.keycloak.quarkus.runtime.configuration.mappers;
+
+import org.keycloak.common.Profile;
+
+import static org.keycloak.config.EventOptions.USER_EVENT_METRICS_ENABLED;
+import static org.keycloak.config.EventOptions.USER_EVENT_METRICS_EVENTS;
+import static org.keycloak.config.EventOptions.USER_EVENT_METRICS_TAGS;
+import static org.keycloak.quarkus.runtime.configuration.Configuration.isTrue;
+import static org.keycloak.quarkus.runtime.configuration.mappers.MetricsPropertyMappers.METRICS_ENABLED_MSG;
+import static org.keycloak.quarkus.runtime.configuration.mappers.MetricsPropertyMappers.metricsEnabled;
+import static org.keycloak.quarkus.runtime.configuration.mappers.PropertyMapper.fromOption;
+
+
+final class EventPropertyMappers {
+
+    private EventPropertyMappers(){}
+
+    public static PropertyMapper<?>[] getMetricsPropertyMappers() {
+        return new PropertyMapper[] {
+                fromOption(USER_EVENT_METRICS_ENABLED)
+                        .to("kc.spi-events-listener-micrometer-user-event-metrics-enabled")
+                        .isEnabled(EventPropertyMappers::userEventsMetricsEnabled, METRICS_ENABLED_MSG + " and feature " + Profile.Feature.USER_EVENT_METRICS.getKey() + " is enabled")
+                        .build(),
+                fromOption(USER_EVENT_METRICS_TAGS)
+                        .to("kc.spi-events-listener-micrometer-user-event-metrics-tags")
+                        .paramLabel("tags")
+                        .isEnabled(EventPropertyMappers::userEventsMetricsTags, "user event metrics are enabled")
+                        .build(),
+                fromOption(USER_EVENT_METRICS_EVENTS)
+                        .to("kc.spi-events-listener-micrometer-user-event-metrics-events")
+                        .paramLabel("events")
+                        .isEnabled(EventPropertyMappers::userEventsMetricsTags, "user event metrics are enabled")
+                        .build(),
+        };
+    }
+
+    private static boolean userEventsMetricsEnabled() {
+        return metricsEnabled() && Profile.isFeatureEnabled(Profile.Feature.USER_EVENT_METRICS);
+    }
+
+    private static boolean userEventsMetricsTags() {
+        return isTrue(USER_EVENT_METRICS_ENABLED);
+    }
+
+}

quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/configuration/mappers/PropertyMappers.java

@@ -59,6 +59,7 @@ public final class PropertyMappers {
         MAPPERS.addAll(ConfigKeystorePropertyMappers.getConfigKeystorePropertyMappers());
         MAPPERS.addAll(ManagementPropertyMappers.getManagementPropertyMappers());
         MAPPERS.addAll(MetricsPropertyMappers.getMetricsPropertyMappers());
+        MAPPERS.addAll(EventPropertyMappers.getMetricsPropertyMappers());
         MAPPERS.addAll(ProxyPropertyMappers.getProxyPropertyMappers());
         MAPPERS.addAll(VaultPropertyMappers.getVaultPropertyMappers());
         MAPPERS.addAll(FeaturePropertyMappers.getMappers());

quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/services/metrics/events/MicrometerUserEventMetricsEventListenerProvider.java

@@ -0,0 +1,151 @@
+/*
+ * Copyright 2024 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.quarkus.runtime.services.metrics.events;
+
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.Metrics;
+import io.micrometer.core.instrument.Tag;
+import io.micrometer.core.instrument.Tags;
+import io.micrometer.core.instrument.binder.BaseUnits;
+import org.jboss.logging.Logger;
+import org.keycloak.events.Details;
+import org.keycloak.events.Errors;
+import org.keycloak.events.Event;
+import org.keycloak.events.EventListenerProvider;
+import org.keycloak.events.EventListenerTransaction;
+import org.keycloak.events.EventType;
+import org.keycloak.events.admin.AdminEvent;
+import org.keycloak.models.KeycloakSession;
+
+import java.util.HashSet;
+import java.util.Locale;
+
+public class MicrometerUserEventMetricsEventListenerProvider implements EventListenerProvider {
+
+    private static final Logger logger = Logger.getLogger(MicrometerUserEventMetricsEventListenerProvider.class);
+
+    private static final String REALM_TAG = "realm";
+    private static final String IDP_TAG = "idp";
+    private static final String CLIENT_ID_TAG = "client.id";
+    private static final String ERROR_TAG = "error";
+    private static final String EVENT_TAG = "event";
+    private static final String DESCRIPTION_OF_EVENT_METER = "Keycloak user events";
+    // Micrometer naming convention that separates lowercase words with a . (dot) character.
+    private static final String KEYCLOAK_METER_NAME_PREFIX = "keycloak.";
+    private static final String USER_EVENTS_METER_NAME = KEYCLOAK_METER_NAME_PREFIX + "user";
+
+    private final boolean withIdp;
+    private final boolean withRealm;
+    private final boolean withClientId;
+    private final HashSet<String> events;
+
+    private final EventListenerTransaction tx =
+            new EventListenerTransaction(null, this::countEvent);
+
+    public MicrometerUserEventMetricsEventListenerProvider(KeycloakSession session, boolean withIdp, boolean withRealm, boolean withClientId, HashSet<String> events) {
+        this.withIdp = withIdp;
+        this.withRealm = withRealm;
+        this.withClientId = withClientId;
+        this.events = events;
+        session.getTransactionManager().enlistAfterCompletion(tx);
+    }
+
+    @Override
+    public void onEvent(Event event) {
+        tx.addEvent(event);
+    }
+
+    private void countEvent(Event event) {
+        logger.debugf("Received user event of type %s in realm %s",
+                event.getType().name(), event.getRealmName());
+
+        String eventTag = format(event.getType());
+        if (events != null && !events.contains(eventTag)) {
+            return;
+        }
+
+        Counter.Builder counterBuilder = Counter.builder(USER_EVENTS_METER_NAME)
+                .description(DESCRIPTION_OF_EVENT_METER)
+                .tags(Tags.of(Tag.of(EVENT_TAG, eventTag),
+                        Tag.of(ERROR_TAG, getError(event))))
+                .baseUnit(BaseUnits.EVENTS);
+
+        if (withRealm) {
+            counterBuilder.tag(REALM_TAG, nullToEmpty(event.getRealmName()));
+        }
+
+        if (withIdp) {
+            counterBuilder.tag(IDP_TAG, getIdentityProvider(event));
+        }
+
+        if (withClientId) {
+            counterBuilder.tag(CLIENT_ID_TAG, getClientId(event));
+        }
+
+        counterBuilder.register(Metrics.globalRegistry)
+                .increment();
+    }
+
+    @Override
+    public void onEvent(AdminEvent event, boolean includeRepresentation) {
+        // do nothing for now
+    }
+
+    private String getIdentityProvider(Event event) {
+        String identityProvider = null;
+        if (event.getDetails() != null) {
+            identityProvider = event.getDetails().get(Details.IDENTITY_PROVIDER);
+        }
+        return nullToEmpty(identityProvider);
+    }
+
+
+    private String getClientId(Event event) {
+        // Don't use the clientId as a tag value of the event CLIENT_NOT_FOUND as it would lead to a metrics cardinality explosion
+        return nullToEmpty(Errors.CLIENT_NOT_FOUND.equals(event.getError())
+                ? "unknown" : event.getClientId());
+    }
+
+    private String getError(Event event) {
+        String error = event.getError();
+        if (error == null && event.getType().name().endsWith("_ERROR")) {
+            error = "unknown";
+        }
+        return nullToEmpty(error);
+    }
+
+    private String nullToEmpty(String value) {
+        return value == null ? "" : value;
+    }
+
+
+    public static String format(EventType type) {
+        // Remove the error suffix so that all events have the same tag.
+        // In dashboards, we can distinguish errors from non-errors by looking at the error tag.
+        String name = type.name();
+        if (name.endsWith("_ERROR")) {
+            name = name.substring(0, name.length() - "_ERROR".length());
+        }
+        return name.toLowerCase(Locale.ROOT);
+    }
+
+    @Override
+    public void close() {
+        // unused
+    }
+}

quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/services/metrics/events/MicrometerUserEventMetricsEventListenerProviderFactory.java

@@ -0,0 +1,94 @@
+/*
+ * Copyright 2024 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.quarkus.runtime.services.metrics.events;
+
+import org.bouncycastle.util.Strings;
+import org.keycloak.Config;
+import org.keycloak.events.EventListenerProvider;
+import org.keycloak.events.EventListenerProviderFactory;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.KeycloakSessionFactory;
+import org.keycloak.provider.EnvironmentDependentProviderFactory;
+
+import java.util.HashSet;
+
+public class MicrometerUserEventMetricsEventListenerProviderFactory implements EventListenerProviderFactory, EnvironmentDependentProviderFactory {
+
+    private static final String ID = "micrometer-user-event-metrics";
+    private static final String TAGS_OPTION = "tags";
+    private static final String EVENTS_OPTION = "events";
+
+    private boolean withIdp, withRealm, withClientId;
+
+    private HashSet<String> events;
+
+    @Override
+    public EventListenerProvider create(KeycloakSession session) {
+        return new MicrometerUserEventMetricsEventListenerProvider(session, withIdp, withRealm, withClientId, events);
+    }
+
+    @Override
+    public void init(Config.Scope config) {
+        String tagsConfig = config.get(TAGS_OPTION);
+        if (tagsConfig != null) {
+            for (String s : Strings.split(tagsConfig, ',')) {
+                switch (s.trim()) {
+                    case "idp" -> withIdp = true;
+                    case "realm" -> withRealm = true;
+                    case "clientId" -> withClientId = true;
+                    default -> throw new IllegalArgumentException("Unknown tag for collecting user event metrics: '" + s + "'");
+                }
+            }
+        }
+        String eventsConfig = config.get(EVENTS_OPTION);
+        if (eventsConfig != null && !eventsConfig.trim().isEmpty()) {
+            events = new HashSet<>();
+            for (String s : Strings.split(eventsConfig, ',')) {
+                events.add(s.trim());
+            }
+        }
+
+    }
+
+    @Override
+    public void postInit(KeycloakSessionFactory factory) {
+    }
+
+    @Override
+    public void close() {
+        // nothing to do
+    }
+
+    @Override
+    public String getId() {
+        return ID;
+    }
+
+
+    @Override
+    public boolean isGlobal() {
+        return true;
+    }
+
+    @Override
+    public boolean isSupported(Config.Scope config) {
+        Boolean enabled = config.getBoolean("enabled");
+        return enabled != null && enabled;
+    }
+
+}

quarkus/runtime/src/main/resources/META-INF/services/org.keycloak.events.EventListenerProviderFactory

@@ -0,0 +1,18 @@
+#
+# Copyright 2016 Red Hat, Inc. and/or its affiliates
+# and other contributors as indicated by the @author tags.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.keycloak.quarkus.runtime.services.metrics.events.MicrometerUserEventMetricsEventListenerProviderFactory

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/MetricsDistTest.java

@@ -17,6 +17,7 @@
 
 package org.keycloak.it.cli.dist;
 
+import static io.restassured.RestAssured.given;
 import static io.restassured.RestAssured.when;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.not;
@@ -80,6 +81,40 @@ public class MetricsDistTest {
 
     }
 
+    @Test
+    @Launch({ "start-dev", "--metrics-enabled=true", "--features=user-event-metrics", "--event-metrics-user-enabled=true" })
+    void testMetricsEndpointWithUserEventMetrics(KeycloakDistribution distribution) {
+        runClientCredentialGrantWithUnknownClientId(distribution);
+
+        distribution.setRequestPort(9000);
+        when().get("/metrics").then()
+                .statusCode(200)
+                .body(containsString("keycloak_user_events_total{error=\"client_not_found\",event=\"client_login\",realm=\"master\"}"));
+
+    }
+
+    @Test
+    @Launch({ "start-dev", "--metrics-enabled=true", "--features=user-event-metrics", "--event-metrics-user-enabled=false" })
+    void testMetricsEndpointWithoutUserEventMetrics(KeycloakDistribution distribution) {
+        runClientCredentialGrantWithUnknownClientId(distribution);
+
+        distribution.setRequestPort(9000);
+        when().get("/metrics").then()
+                .statusCode(200)
+                .body(not(containsString("keycloak_user_events_total{error=\"client_not_found\",event=\"client_login\",realm=\"master\"}")));
+
+    }
+
+    private static void runClientCredentialGrantWithUnknownClientId(KeycloakDistribution distribution) {
+        distribution.setRequestPort(8080);
+        given().formParam("grant_type", "client_credentials")
+                .formParam("client_id", "unknown")
+                .formParam("client_secret", "unknown").
+                when().post("/realms/master/protocol/openid-connect/token")
+                .then()
+                .statusCode(401);
+    }
+
     @Test
     void testUsingRelativePath(KeycloakDistribution distribution) {
         for (String relativePath : List.of("/auth", "/auth/", "auth")) {

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testExportHelpAll.approved.txt

@@ -241,6 +241,21 @@ Tracing (Preview):
                        defined in the 'tracing-resource-attributes' property. Default: keycloak.
                        Available only when 'opentelemetry' feature and Tracing is enabled.
 
+Events (Preview):
+
+--event-metrics-user-enabled <true|false>
+                     Preview: Create metrics based on user events. Default: false. Available only
+                       when metrics are enabled and feature user-event-metrics is enabled.
+--event-metrics-user-events <events>
+                     Preview: Comma-separated list of events to be collected for user event
+                       metrics. Reduce the number of metrics. If empty or not set, all events
+                       create a metric. Available only when user event metrics are enabled.
+--event-metrics-user-tags <tags>
+                     Preview: Comma-separated list of tags to be collected for user event metrics.
+                       By default only 'realm' is enabled to avoid a high metrics cardinality.
+                       Possible values are: realm, idp, clientId. Default: realm. Available only
+                       when user event metrics are enabled.
+
 Truststore:
 
 --tls-hostname-verifier <tls-hostname-verifier>

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testImportHelpAll.approved.txt

@@ -241,6 +241,21 @@ Tracing (Preview):
                        defined in the 'tracing-resource-attributes' property. Default: keycloak.
                        Available only when 'opentelemetry' feature and Tracing is enabled.
 
+Events (Preview):
+
+--event-metrics-user-enabled <true|false>
+                     Preview: Create metrics based on user events. Default: false. Available only
+                       when metrics are enabled and feature user-event-metrics is enabled.
+--event-metrics-user-events <events>
+                     Preview: Comma-separated list of events to be collected for user event
+                       metrics. Reduce the number of metrics. If empty or not set, all events
+                       create a metric. Available only when user event metrics are enabled.
+--event-metrics-user-tags <tags>
+                     Preview: Comma-separated list of tags to be collected for user event metrics.
+                       By default only 'realm' is enabled to avoid a high metrics cardinality.
+                       Possible values are: realm, idp, clientId. Default: realm. Available only
+                       when user event metrics are enabled.
+
 Truststore:
 
 --tls-hostname-verifier <tls-hostname-verifier>

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testStartDevHelpAll.approved.txt

@@ -449,6 +449,21 @@ Tracing (Preview):
                        defined in the 'tracing-resource-attributes' property. Default: keycloak.
                        Available only when 'opentelemetry' feature and Tracing is enabled.
 
+Events (Preview):
+
+--event-metrics-user-enabled <true|false>
+                     Preview: Create metrics based on user events. Default: false. Available only
+                       when metrics are enabled and feature user-event-metrics is enabled.
+--event-metrics-user-events <events>
+                     Preview: Comma-separated list of events to be collected for user event
+                       metrics. Reduce the number of metrics. If empty or not set, all events
+                       create a metric. Available only when user event metrics are enabled.
+--event-metrics-user-tags <tags>
+                     Preview: Comma-separated list of tags to be collected for user event metrics.
+                       By default only 'realm' is enabled to avoid a high metrics cardinality.
+                       Possible values are: realm, idp, clientId. Default: realm. Available only
+                       when user event metrics are enabled.
+
 Truststore:
 
 --tls-hostname-verifier <tls-hostname-verifier>

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testStartHelpAll.approved.txt

@@ -450,6 +450,21 @@ Tracing (Preview):
                        defined in the 'tracing-resource-attributes' property. Default: keycloak.
                        Available only when 'opentelemetry' feature and Tracing is enabled.
 
+Events (Preview):
+
+--event-metrics-user-enabled <true|false>
+                     Preview: Create metrics based on user events. Default: false. Available only
+                       when metrics are enabled and feature user-event-metrics is enabled.
+--event-metrics-user-events <events>
+                     Preview: Comma-separated list of events to be collected for user event
+                       metrics. Reduce the number of metrics. If empty or not set, all events
+                       create a metric. Available only when user event metrics are enabled.
+--event-metrics-user-tags <tags>
+                     Preview: Comma-separated list of tags to be collected for user event metrics.
+                       By default only 'realm' is enabled to avoid a high metrics cardinality.
+                       Possible values are: realm, idp, clientId. Default: realm. Available only
+                       when user event metrics are enabled.
+
 Truststore:
 
 --tls-hostname-verifier <tls-hostname-verifier>

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testStartOptimizedHelpAll.approved.txt

@@ -390,6 +390,18 @@ Tracing (Preview):
                        defined in the 'tracing-resource-attributes' property. Default: keycloak.
                        Available only when 'opentelemetry' feature and Tracing is enabled.
 
+Events (Preview):
+
+--event-metrics-user-events <events>
+                     Preview: Comma-separated list of events to be collected for user event
+                       metrics. Reduce the number of metrics. If empty or not set, all events
+                       create a metric. Available only when user event metrics are enabled.
+--event-metrics-user-tags <tags>
+                     Preview: Comma-separated list of tags to be collected for user event metrics.
+                       By default only 'realm' is enabled to avoid a high metrics cardinality.
+                       Possible values are: realm, idp, clientId. Default: realm. Available only
+                       when user event metrics are enabled.
+
 Truststore:
 
 --tls-hostname-verifier <tls-hostname-verifier>

server-spi-private/src/main/java/org/keycloak/events/EventBuilder.java

@@ -22,7 +22,6 @@ import org.keycloak.common.ClientConnection;
 import org.keycloak.common.util.Time;
 import org.keycloak.models.ClientModel;
 import org.keycloak.models.KeycloakSession;
-import org.keycloak.models.KeycloakSessionFactory;
 import org.keycloak.models.RealmModel;
 import org.keycloak.models.UserModel;
 import org.keycloak.models.UserSessionModel;
@@ -30,6 +29,7 @@ import org.keycloak.models.UserSessionModel;
 import org.keycloak.models.utils.KeycloakModelUtils;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
@@ -80,17 +80,18 @@ public class EventBuilder {
     }
 
     private static List<EventListenerProvider> getEventListeners(KeycloakSession session, RealmModel realm) {
-        return realm.getEventsListenersStream().map(id -> {
-            EventListenerProvider listener = session.getProvider(EventListenerProvider.class, id);
-            if (listener != null) {
-                return listener;
-            } else {
-                log.error("Event listener '" + id + "' registered, but provider not found");
-                return null;
-            }
-        })
-        .filter(Objects::nonNull)
-        .collect(Collectors.toList());
+        HashSet<String> realmListeners = new HashSet<>(realm.getEventsListenersStream().toList());
+        List<EventListenerProvider> result = session.getKeycloakSessionFactory().getProviderFactoriesStream(EventListenerProvider.class)
+                .filter(providerFactory -> realmListeners.contains(providerFactory.getId()) || ((EventListenerProviderFactory) providerFactory).isGlobal())
+                .map(providerFactory -> {
+                    realmListeners.remove(providerFactory.getId());
+                    return session.getProvider(EventListenerProvider.class, providerFactory.getId());
+                })
+                .toList();
+        if (!realmListeners.isEmpty()) {
+            log.error("Event listeners " + realmListeners + " registered, but provider not found");
+        }
+        return result;
     }
 
     private EventBuilder(KeycloakSession session, EventStoreProvider store, List<EventListenerProvider> listeners, RealmModel realm, Event event) {
@@ -159,10 +160,10 @@ public class EventBuilder {
         event.getDetails().put(key, value);
         return this;
     }
-    
+
     /**
-     * Add event detail where strings from the input Collection are filtered not to contain <code>null</code> and then joined using <code>::</code> character. 
-     * 
+     * Add event detail where strings from the input Collection are filtered not to contain <code>null</code> and then joined using <code>::</code> character.
+     *
      * @param key of the detail
      * @param values, can be null
      * @return builder for chaining
@@ -173,10 +174,10 @@ public class EventBuilder {
         }
         return detail(key, values.stream().filter(Objects::nonNull).collect(Collectors.joining("::")));
     }
-    
+
     /**
-     * Add event detail where strings from the input Stream are filtered not to contain <code>null</code> and then joined using <code>::</code> character. 
-     * 
+     * Add event detail where strings from the input Stream are filtered not to contain <code>null</code> and then joined using <code>::</code> character.
+     *
      * @param key of the detail
      * @param values, can be null
      * @return builder for chaining

server-spi-private/src/main/java/org/keycloak/events/EventListenerProviderFactory.java

@@ -24,4 +24,8 @@ import org.keycloak.provider.ProviderFactory;
  */
 public interface EventListenerProviderFactory extends ProviderFactory<EventListenerProvider> {
 
+    default boolean isGlobal() {
+        return false;
+    }
+
 }

services/src/main/java/org/keycloak/services/resources/admin/AdminEventBuilder.java

@@ -22,6 +22,7 @@ import org.jboss.logging.Logger;
 import org.keycloak.common.ClientConnection;
 import org.keycloak.common.util.Time;
 import org.keycloak.events.EventListenerProvider;
+import org.keycloak.events.EventListenerProviderFactory;
 import org.keycloak.events.EventStoreProvider;
 import org.keycloak.events.admin.AdminEvent;
 import org.keycloak.events.admin.AuthDetails;
@@ -31,17 +32,15 @@ import org.keycloak.models.ClientModel;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.RealmModel;
 import org.keycloak.models.UserModel;
-import org.keycloak.models.utils.StripSecretsUtils;
-import org.keycloak.representations.idm.UserRepresentation;
 import org.keycloak.services.ServicesLogger;
 import org.keycloak.util.JsonSerialization;
 
 import jakarta.ws.rs.core.UriInfo;
 import java.io.IOException;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.UUID;
-import java.util.function.Predicate;
 
 public class AdminEventBuilder {
 
@@ -130,16 +129,16 @@ public class AdminEventBuilder {
     }
 
     private AdminEventBuilder addListeners(KeycloakSession session) {
-        realm.getEventsListenersStream()
-                .filter(((Predicate<String>) listeners::containsKey).negate())
-                .forEach(id -> {
-                    EventListenerProvider listener = session.getProvider(EventListenerProvider.class, id);
-                    if (listener != null) {
-                        listeners.put(id, listener);
-                    } else {
-                        ServicesLogger.LOGGER.providerNotFound(id);
+        HashSet<String> realmListeners = new HashSet<>(realm.getEventsListenersStream().toList());
+        session.getKeycloakSessionFactory().getProviderFactoriesStream(EventListenerProvider.class)
+                .filter(providerFactory -> realmListeners.contains(providerFactory.getId()) || ((EventListenerProviderFactory) providerFactory).isGlobal())
+                .forEach(providerFactory -> {
+                    realmListeners.remove(providerFactory.getId());
+                    if (!listeners.containsKey(providerFactory.getId())) {
+                        listeners.put(providerFactory.getId(), ((EventListenerProviderFactory) providerFactory).create(session));
                     }
                 });
+        realmListeners.forEach(ServicesLogger.LOGGER::providerNotFound);
         return this;
     }
 

services/src/main/java/org/keycloak/services/resources/admin/RealmsAdminResource.java

@@ -34,7 +34,6 @@ import org.keycloak.models.ModelException;
 import org.keycloak.models.ModelIllegalStateException;
 import org.keycloak.models.RealmModel;
 import org.keycloak.models.utils.ModelToRepresentation;
-import org.keycloak.models.utils.StripSecretsUtils;
 import org.keycloak.policy.PasswordPolicyNotMetException;
 import org.keycloak.protocol.oidc.TokenManager;
 import org.keycloak.representations.idm.RealmRepresentation;
@@ -190,9 +189,7 @@ public class RealmsAdminResource {
     /**
      * Base path for the admin REST API for one particular realm.
      *
-     * @param headers
      * @param name realm name (not id!)
-     * @return
      */
     @Path("{realm}")
     public RealmAdminResource getRealmAdmin(@PathParam("realm") @Parameter(description = "realm name (not id!)") final String name) {
@@ -206,8 +203,8 @@ public class RealmsAdminResource {
         }
         AdminPermissionEvaluator realmAuth = AdminPermissions.evaluator(session, realm, auth);
 
-        AdminEventBuilder adminEvent = new AdminEventBuilder(realm, auth, session, clientConnection);
         session.getContext().setRealm(realm);
+        AdminEventBuilder adminEvent = new AdminEventBuilder(realm, auth, session, clientConnection);
 
         return new RealmAdminResource(session, realmAuth, adminEvent);
     }

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/events/EventMetricsProviderTest.java

@@ -0,0 +1,165 @@
+/*
+ * Copyright 2024 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.testsuite.events;
+
+import io.micrometer.core.instrument.Metrics;
+import org.hamcrest.MatcherAssert;
+import org.hamcrest.Matchers;
+import org.junit.After;
+import org.junit.Test;
+import org.keycloak.common.Profile;
+import org.keycloak.events.Details;
+import org.keycloak.events.Errors;
+import org.keycloak.events.EventBuilder;
+import org.keycloak.events.EventType;
+import org.keycloak.models.RealmModel;
+import org.keycloak.representations.idm.RealmRepresentation;
+import org.keycloak.testsuite.AbstractKeycloakTest;
+import org.keycloak.testsuite.arquillian.annotation.EnableFeature;
+import org.keycloak.testsuite.arquillian.containers.AbstractQuarkusDeployableContainer;
+import org.keycloak.testsuite.util.RealmBuilder;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.keycloak.testsuite.auth.page.AuthRealm.TEST;
+
+/**
+ * @author aschwart
+ */
+@EnableFeature(Profile.Feature.USER_EVENT_METRICS)
+public class EventMetricsProviderTest extends AbstractKeycloakTest {
+
+    private final static String CLIENT_ID = "CLIENT_ID";
+
+    private void enableUserEventMetrics(String tags, String events) {
+        try {
+            suiteContext.getAuthServerInfo().getArquillianContainer().getDeployableContainer().stop();
+            enableEventMetricsOptions(tags, events);
+            suiteContext.getAuthServerInfo().getArquillianContainer().getDeployableContainer().start();
+            reconnectAdminClient();
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private void enableEventMetricsOptions(String tags, String events) {
+        if (suiteContext.getAuthServerInfo().isQuarkus()) {
+            AbstractQuarkusDeployableContainer container = (AbstractQuarkusDeployableContainer) suiteContext.getAuthServerInfo().getArquillianContainer().getDeployableContainer();
+            List<String> args = new ArrayList<>();
+            args.add("--metrics-enabled=true");
+            args.add("--event-metrics-user-enabled=true");
+            if (tags != null) {
+                args.add("--event-metrics-user-tags=" + tags);
+            }
+            if (events != null) {
+                args.add("--event-metrics-user-events=" + events);
+            }
+            container.setAdditionalBuildArgs(args);
+        }
+        else {
+            setConfigProperty("keycloak.metrics-enabled", "true");
+            setConfigProperty("keycloak.event-metrics-user-enabled", "true");
+        }
+    }
+
+    private static void setConfigProperty(String name, String value) {
+        if (value != null) {
+            System.setProperty(name, value);
+        }
+        else {
+            System.clearProperty(name);
+        }
+    }
+
+    @Override
+    public void addTestRealms(List<RealmRepresentation> testRealms) {
+        RealmBuilder realm = RealmBuilder.create().name(TEST);
+        testRealms.add(realm.build());
+    }
+
+    @Test
+    public void shouldCountSingleEventWithTagsAndFilter() {
+        enableUserEventMetrics(null, null);
+
+        testingClient.server().run(session -> {
+            RealmModel realm = session.realms().getRealmByName(TEST);
+            session.getContext().setRealm(realm);
+            EventBuilder eventBuilder = new EventBuilder(realm, session);
+            eventBuilder.event(EventType.LOGIN)
+                            .client(CLIENT_ID);
+            eventBuilder.success();
+        });
+
+        testingClient.server().run(session -> {
+            MatcherAssert.assertThat("Searching for metrics match in " + Metrics.globalRegistry.find("keycloak.user").meters(),
+                    Metrics.globalRegistry.counter("keycloak.user", "event", "login", "error", "", "realm", TEST).count() == 1);
+        });
+
+        // Show all labels, but filter out events like logout@
+        enableUserEventMetrics("realm,idp,clientId", "login,refresh_token");
+
+        testingClient.server().run(session -> {
+            RealmModel realm = session.realms().getRealmByName(TEST);
+            session.getContext().setRealm(realm);
+
+            // this event is not recorded as a metric as the event is not listed in the configuration
+            EventBuilder eventBuilder = new EventBuilder(realm, session);
+            eventBuilder.event(EventType.LOGOUT)
+                    .client(CLIENT_ID)
+                    .detail(Details.IDENTITY_PROVIDER, "IDENTITY_PROVIDER");
+            eventBuilder.success();
+
+            // this event is recorded as an error
+            eventBuilder = new EventBuilder(realm, session);
+            eventBuilder.event(EventType.LOGIN)
+                    .client(CLIENT_ID)
+                    .detail(Details.IDENTITY_PROVIDER, "IDENTITY_PROVIDER");
+            eventBuilder.error("ERROR");
+
+            // this event is recorded with the special logic about not found clients
+            eventBuilder = new EventBuilder(realm, session);
+            eventBuilder.event(EventType.REFRESH_TOKEN)
+                    .client(CLIENT_ID);
+            eventBuilder.error(Errors.CLIENT_NOT_FOUND);
+
+        });
+
+        testingClient.server().run(session -> {
+            MatcherAssert.assertThat("Two metrics recorded",
+                    Metrics.globalRegistry.find("keycloak.user").meters().size(), Matchers.equalTo(2));
+            MatcherAssert.assertThat("Searching for login error metric",
+                    Metrics.globalRegistry.counter("keycloak.user", "event", "login", "error", "ERROR", "realm", TEST, "client.id", CLIENT_ID, "idp", "IDENTITY_PROVIDER").count() == 1);
+            MatcherAssert.assertThat("Searching for refresh with unknown client",
+                    Metrics.globalRegistry.counter("keycloak.user", "event", "refresh_token", "error", "client_not_found", "realm", TEST, "client.id", "unknown", "idp", "").count() == 1);
+        });
+
+    }
+
+    @After
+    public void resetHostnameSettings() {
+        if (suiteContext.getAuthServerInfo().isQuarkus()) {
+            AbstractQuarkusDeployableContainer container = (AbstractQuarkusDeployableContainer) suiteContext.getAuthServerInfo().getArquillianContainer().getDeployableContainer();
+            container.resetConfiguration();
+        } else {
+            setConfigProperty("keycloak.metrics-enabled", null);
+            setConfigProperty("keycloak.event-metrics-user-enabled", null);
+        }
+    }
+
+}