KEYCLOAK-13639. Added metrics and custom healthcheck endpoints, both enabled via 'metrics.enabled' config parameter.

quarkus/deployment/pom.xml

@@ -66,6 +66,14 @@
             <groupId>io.quarkus</groupId>
             <artifactId>quarkus-bootstrap-core</artifactId>
         </dependency>
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-smallrye-health-deployment</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-smallrye-metrics-deployment</artifactId>
+        </dependency>
         <dependency>
             <groupId>io.quarkus</groupId>
             <artifactId>quarkus-junit5-internal</artifactId>

quarkus/deployment/src/main/java/org/keycloak/quarkus/deployment/KeycloakProcessor.java

@@ -46,6 +46,7 @@ import org.keycloak.common.Profile;
 import org.keycloak.config.ConfigProviderFactory;
 import org.keycloak.configuration.Configuration;
 import org.keycloak.configuration.KeycloakConfigSourceProvider;
+import org.keycloak.configuration.MicroProfileConfigProvider;
 import org.keycloak.connections.jpa.DefaultJpaConnectionProviderFactory;
 import org.keycloak.connections.jpa.updater.liquibase.LiquibaseJpaUpdaterProviderFactory;
 import org.keycloak.connections.jpa.updater.liquibase.conn.DefaultLiquibaseConnectionProvider;
@@ -194,9 +195,13 @@ class KeycloakProcessor {
         indexDependencyBuildItemBuildProducer.produce(new IndexDependencyBuildItem("org.keycloak", "keycloak-services"));
     }
 
+    @Record(ExecutionTime.RUNTIME_INIT)
     @BuildStep
-    void initializeRouter(BuildProducer<FilterBuildItem> routes) {
+    void initializeFilter(BuildProducer<FilterBuildItem> routes, KeycloakRecorder recorder) {
-        routes.produce(new FilterBuildItem(new QuarkusRequestFilter(), FilterBuildItem.AUTHORIZATION - 10));
+        Optional<Boolean> metricsEnabled = Configuration.getOptionalBooleanValue(MicroProfileConfigProvider.NS_KEYCLOAK_PREFIX.concat("metrics.enabled"));
+
+        routes.produce(new FilterBuildItem(recorder.createFilter(metricsEnabled.orElse(false)),
+                FilterBuildItem.AUTHORIZATION - 10));
     }
 
     @BuildStep(onlyIf = IsDevelopment.class)

quarkus/runtime/pom.xml

@@ -63,6 +63,14 @@
             <groupId>io.quarkus</groupId>
             <artifactId>quarkus-core</artifactId>
         </dependency>
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-smallrye-health</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-smallrye-metrics</artifactId>
+        </dependency>
 
         <!-- CLI -->
         <dependency>

quarkus/runtime/src/main/java/org/keycloak/configuration/Configuration.java

@@ -18,6 +18,7 @@
 package org.keycloak.configuration;
 
 import java.util.Optional;
+import java.util.function.Function;
 
 import io.smallrye.config.ConfigValue;
 import io.smallrye.config.SmallRyeConfig;
@@ -69,4 +70,13 @@ public final class Configuration {
     public static Optional<String> getOptionalValue(String name) {
         return getConfig().getOptionalValue(name, String.class);
     }
+
+    public static Optional<Boolean> getOptionalBooleanValue(String name) {
+        return getConfig().getOptionalValue(name, String.class).map(new Function<String, Boolean>() {
+            @Override
+            public Boolean apply(String s) {
+                return Boolean.parseBoolean(s);
+            }
+        });
+    }
 }

quarkus/runtime/src/main/java/org/keycloak/configuration/PropertyMapper.java

@@ -60,6 +60,10 @@ public class PropertyMapper {
         return MAPPERS.computeIfAbsent(toProperty, s -> new PropertyMapper(fromProperty, s, null, transformer, null, true, description, false));
     }
 
+    static PropertyMapper createBuildTimeProperty(String fromProperty, String toProperty, String description) {
+        return MAPPERS.computeIfAbsent(toProperty, s -> new PropertyMapper(fromProperty, s, null, null, null, true, description, false));
+    }
+
     static Map<String, PropertyMapper> MAPPERS = new HashMap<>();
 
     static PropertyMapper IDENTITY = new PropertyMapper(null, null, null, null, null) {

quarkus/runtime/src/main/java/org/keycloak/configuration/PropertyMappers.java

@@ -46,6 +46,7 @@ public final class PropertyMappers {
         configureProxyMappers();
         configureClustering();
         configureHostnameProviderMappers();
+        configureMetrics();
     }
 
     private static void configureHttpPropertyMappers() {
@@ -150,6 +151,10 @@ public final class PropertyMappers {
         create("hostname-force-backend-url-to-frontend-url ", "kc.spi.hostname.default.force-backend-url-to-frontend-url", "Forces backend requests to go through the URL defined as the frontend-url. Defaults to false. Possible values are true or false.");
     }
 
+    private static void configureMetrics() {
+        createBuildTimeProperty("metrics.enabled", "quarkus.datasource.metrics.enabled", "If the server should expose metrics and healthcheck. If enabled, metrics are available at the '/metrics' endpoint and healthcheck at the '/health' endpoint.");
+    }
+
     static ConfigValue getValue(ConfigSourceInterceptorContext context, String name) {
         return PropertyMapper.MAPPERS.getOrDefault(name, PropertyMapper.IDENTITY)
                 .getOrDefault(name, context, context.proceed(name));

quarkus/runtime/src/main/java/org/keycloak/provider/quarkus/QuarkusRequestFilter.java

@@ -17,8 +17,9 @@
 
 package org.keycloak.provider.quarkus;
 
+import java.util.function.Predicate;
+
 import org.keycloak.common.ClientConnection;
-import org.keycloak.models.KeycloakSession;
 import org.keycloak.services.filters.AbstractRequestFilter;
 
 import io.vertx.core.AsyncResult;
@@ -39,8 +40,15 @@ public class QuarkusRequestFilter extends AbstractRequestFilter implements Handl
         // we don't really care about the result because any exception thrown should be handled by the parent class
     };
 
+    private Predicate<RoutingContext> enabledEndpoints;
+
     @Override
     public void handle(RoutingContext context) {
+        if (!enabledEndpoints.test(context)) {
+            context.fail(404);
+            return;
+        }
+
         // our code should always be run as blocking until we don't provide a better support for running non-blocking code
         // in the event loop
         context.vertx().executeBlocking(promise -> {
@@ -94,4 +102,8 @@ public class QuarkusRequestFilter extends AbstractRequestFilter implements Handl
             }
         };
     }
+
+    public void setEnabledEndpoints(Predicate<RoutingContext> disabledEndpoints) {
+        this.enabledEndpoints = disabledEndpoints;
+    }
 }

quarkus/runtime/src/main/java/org/keycloak/quarkus/KeycloakRecorder.java

@@ -26,6 +26,8 @@ import java.util.function.Predicate;
 import java.util.stream.StreamSupport;
 
 import io.smallrye.config.ConfigValue;
+import io.vertx.core.Handler;
+import io.vertx.ext.web.RoutingContext;
 import org.jboss.logging.Logger;
 import org.keycloak.QuarkusKeycloakSessionFactory;
 import org.keycloak.cli.ShowConfigCommand;
@@ -43,6 +45,7 @@ import org.keycloak.provider.Spi;
 import io.quarkus.runtime.annotations.Recorder;
 import liquibase.logging.LogFactory;
 import liquibase.servicelocator.ServiceLocator;
+import org.keycloak.provider.quarkus.QuarkusRequestFilter;
 import org.keycloak.util.Environment;
 
 @Recorder
@@ -210,4 +213,23 @@ public class KeycloakRecorder {
             }
         });
     }
+
+    public Handler<RoutingContext> createFilter(boolean metricsEnabled) {
+        QuarkusRequestFilter handler = new QuarkusRequestFilter();
+
+        handler.setEnabledEndpoints(new Predicate<RoutingContext>() {
+            @Override
+            public boolean test(RoutingContext context) {
+
+                if (context.request().uri().startsWith("/metrics") ||
+                    context.request().uri().startsWith("/health")) {
+                    return metricsEnabled;
+                }
+
+                return true;
+            }
+        });
+
+        return handler;
+    }
 }

quarkus/runtime/src/main/java/org/keycloak/services/health/KeycloakReadyHealthCheck.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2020 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.services.health;
+
+import io.agroal.api.AgroalDataSource;
+import io.quarkus.agroal.runtime.health.DataSourceHealthCheck;
+import org.eclipse.microprofile.health.HealthCheckResponse;
+import org.eclipse.microprofile.health.HealthCheckResponseBuilder;
+import org.eclipse.microprofile.health.Readiness;
+
+import javax.enterprise.context.ApplicationScoped;
+import javax.inject.Inject;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * Keycloak Healthcheck Readiness Probe.
+ *
+ * Performs a hybrid between the passive and the active mode. If there are no healthy connections in the pool,
+ * it invokes the standard <code>DataSourceHealthCheck</code> that creates a new connection and checks if its valid.
+ *
+ * @see <a href="https://github.com/keycloak/keycloak-community/pull/55">Healthcheck API Design</a>
+ */
+@Readiness
+@ApplicationScoped
+public class KeycloakReadyHealthCheck extends DataSourceHealthCheck {
+
+    /**
+     * Date formatter, the same as used by Quarkus. This enables users to quickly compare the date printed
+     * by the probe with the logs.
+     */
+    static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss,SSS").withZone(ZoneId.systemDefault());
+
+    @Inject
+    AgroalDataSource agroalDataSource;
+
+    AtomicReference<Instant> failingSince = new AtomicReference<>();
+
+    @Override
+    public HealthCheckResponse call() {
+        HealthCheckResponseBuilder builder = HealthCheckResponse.named("Keycloak database connections health check").up();
+        long activeCount = agroalDataSource.getMetrics().activeCount();
+        long invalidCount = agroalDataSource.getMetrics().invalidCount();
+        if (activeCount < 1 || invalidCount > 0) {
+            HealthCheckResponse activeCheckResult = super.call();
+            if (activeCheckResult.getState() == HealthCheckResponse.State.DOWN) {
+                builder.down();
+                Instant failingTime = failingSince.updateAndGet(this::createInstanceIfNeeded);
+                builder.withData("Failing since", DATE_FORMATTER.format(failingTime));
+            }
+        } else {
+            failingSince.set(null);
+        }
+        return builder.build();
+    }
+
+    Instant createInstanceIfNeeded(Instant instant) {
+        if (instant == null) {
+            return Instant.now();
+        }
+        return instant;
+    }
+}

quarkus/server/src/main/resources/META-INF/keycloak.properties

@@ -1,4 +1,4 @@
-# Default and non-production grade database vendor 
+# Default and non-production grade database vendor
 db=h2-file
 
 # Default, and insecure, and non-production grade configuration for the development profile
@@ -7,6 +7,9 @@ db=h2-file
 %dev.db.password = keycloak
 %dev.cluster=local
 
+# Metrics and healthcheck are disabled by default
+metrics.enabled=false
+
 # Logging configuration. INFO is the default level for most of the categories
 #quarkus.log.level = DEBUG
 quarkus.log.category."org.jboss.resteasy.resteasy_jaxrs.i18n".level=WARN

quarkus/server/src/main/resources/application.properties

@@ -7,4 +7,8 @@ quarkus.package.main-class=keycloak
 
 quarkus.http.root-path=/
 quarkus.application.name=Keycloak
-quarkus.banner.enabled=false
+quarkus.banner.enabled=false
+
+# Disable the default data source health check by Agroal extension, since we provide our own (default is true)
+quarkus.datasource.health.enabled=false
+

services/src/main/java/org/keycloak/transaction/JtaTransactionWrapper.java

@@ -88,6 +88,10 @@ public class JtaTransactionWrapper implements KeycloakTransaction {
     @Override
     public void commit() {
         try {
+            if (Status.STATUS_NO_TRANSACTION == tm.getStatus() ||
+                Status.STATUS_ACTIVE != tm.getStatus()) {
+                return;
+            }
             logger.debug("JtaTransactionWrapper  commit");
             tm.commit();
         } catch (Exception e) {
@@ -100,6 +104,10 @@ public class JtaTransactionWrapper implements KeycloakTransaction {
     @Override
     public void rollback() {
         try {
+            if (Status.STATUS_NO_TRANSACTION == tm.getStatus() ||
+                Status.STATUS_ACTIVE != tm.getStatus()) {
+                return;
+            }
             logger.debug("JtaTransactionWrapper rollback");
             tm.rollback();
         } catch (Exception e) {