task: deprecate WILDCARD and STRICT options (#26833)

closes: #24893

Signed-off-by: Steve Hawkins <shawkins@redhat.com>

common/src/main/java/org/keycloak/common/enums/HostnameVerificationPolicy.java

@@ -25,12 +25,19 @@ public enum HostnameVerificationPolicy {
     ANY,
 
     /**
-     * Allows wildcards in subdomain names i.e. *.foo.com
+     * Allows wildcards in subdomain names (e.g. *.foo.com) to match anything, including multiple levels (e.g. a.b.foo.com)
      */
+    @Deprecated
     WILDCARD,
 
     /**
      * CN must match hostname connecting to
      */
-    STRICT
+    @Deprecated
+    STRICT,
+
+    /**
+     * Similar to STRICT, but checks against a more complete public suffix matcher
+     */
+    DEFAULT
 }

docs/documentation/upgrading/topics/changes/changes-25_0_0.adoc

@@ -35,6 +35,18 @@ Users can configure this limit by setting the provider configuration option `spi
 bin/kc.[sh|bat] --spi-connections-http-client-default-max-consumed-response-size=1000000
 ----
 
+= Hostname Verification Policy
+
+The default for `spi-truststore-file-hostname-verification-policy` and the new `tls-hostname-verifier` option is now DEFAULT, rather than WILDCARD. The WILDCARD and STRICT option values have been deprecated - you should simply rely upon DEFAULT instead. 
+
+Behavior supported by WILDCARD, that is not supported by DEFAULT:
+* allows wildcards in subdomain names (e.g. *.foo.com) to match anything, including multiple levels (e.g. a.b.foo.com). 
+* allows matching against well known public suffixes - e.g. foo.co.gl may match *.co.gl
+
+Behavior supported by STRICT, that is not supported by DEFAULT:
+* STRICT uses a small exclusion list for 2 or 3 letter domain names ending in a 2 letter top level (*.XXX.YY) when determining if a wildcard matches. Instead DEFAULT uses a more complete list of public suffix rules and exclusions from https://publicsuffix.org/list/
+
+It is not expected that you should be relying upon these behaviors from the WILDCARD or STRICT options.
 
 = Removed a model module
 

docs/guides/server/keycloak-truststore.adoc

@@ -28,9 +28,10 @@ It is still possible to directly set your own `javax.net.ssl` truststore System
 
 You may refine how hostnames are verified by TLS connections with the `tls-hostname-verifier` property.
 
-* `WILDCARD` (the default) allows wildcards in subdomain names, such as *.foo.com.
+* `DEFAULT` (the default) allows wildcards in subdomain names (e.g. *.foo.com) to match names with the same number of levels (e.g. a.foo.com, but not a.b.foo.com) - with rules and exclusions for public suffixes based upon https://publicsuffix.org/list/
 * `ANY` means that the hostname is not verified.
-* When using `STRICT`, the Common Name (CN) must match the hostname exactly.
+* `WILDCARD` (deprecated) allows wildcards in subdomain names (e.g. *.foo.com) to match anything, including multiple levels (e.g. a.b.foo.com). Use DEFAULT instead.
+* `STRICT` (deprecated) allows wildcards in subdomain names (e.g. *.foo.com) to match names with the same number of levels (e.g. a.foo.com, but not a.b.foo.com) - with some limited exclusions. Use DEFAULT instead.
 +
 Please note that this setting does not apply to LDAP secure connections, which require strict hostname checking.
 

quarkus/config-api/src/main/java/org/keycloak/config/TruststoreOptions.java

@@ -3,6 +3,7 @@ package org.keycloak.config;
 import org.keycloak.common.enums.HostnameVerificationPolicy;
 
 import java.util.List;
+import java.util.Set;
 
 public class TruststoreOptions {
 
@@ -14,7 +15,8 @@ public class TruststoreOptions {
     public static final Option<HostnameVerificationPolicy> HOSTNAME_VERIFICATION_POLICY = new OptionBuilder<>("tls-hostname-verifier", HostnameVerificationPolicy.class)
             .category(OptionCategory.TRUSTSTORE)
             .description("The TLS hostname verification policy for out-going HTTPS and SMTP requests.")
-            .defaultValue(HostnameVerificationPolicy.WILDCARD)
+            .defaultValue(HostnameVerificationPolicy.DEFAULT)
+            .deprecatedValues(Set.of("STRICT", "WILDCARD"), "STRICT and WILDCARD have been deprecated, use DEFAULT instead.")
             .build();
 
 }

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testExportHelp.unix.approved.txt

@@ -92,7 +92,8 @@ Truststore:
 
 --tls-hostname-verifier <tls-hostname-verifier>
                      The TLS hostname verification policy for out-going HTTPS and SMTP requests.
-                       Possible values are: ANY, WILDCARD, STRICT. Default: WILDCARD.
+                       Possible values are: ANY, WILDCARD (deprecated), STRICT (deprecated),
+                       DEFAULT. Default: DEFAULT.
 --truststore-paths <truststore-paths>
                      List of pkcs12 (p12 or pfx file extensions), PEM files, or directories
                        containing those files that will be used as a system truststore.

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testExportHelpAll.unix.approved.txt

@@ -137,7 +137,8 @@ Truststore:
 
 --tls-hostname-verifier <tls-hostname-verifier>
                      The TLS hostname verification policy for out-going HTTPS and SMTP requests.
-                       Possible values are: ANY, WILDCARD, STRICT. Default: WILDCARD.
+                       Possible values are: ANY, WILDCARD (deprecated), STRICT (deprecated),
+                       DEFAULT. Default: DEFAULT.
 --truststore-paths <truststore-paths>
                      List of pkcs12 (p12 or pfx file extensions), PEM files, or directories
                        containing those files that will be used as a system truststore.

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testImportHelp.unix.approved.txt

@@ -92,7 +92,8 @@ Truststore:
 
 --tls-hostname-verifier <tls-hostname-verifier>
                      The TLS hostname verification policy for out-going HTTPS and SMTP requests.
-                       Possible values are: ANY, WILDCARD, STRICT. Default: WILDCARD.
+                       Possible values are: ANY, WILDCARD (deprecated), STRICT (deprecated),
+                       DEFAULT. Default: DEFAULT.
 --truststore-paths <truststore-paths>
                      List of pkcs12 (p12 or pfx file extensions), PEM files, or directories
                        containing those files that will be used as a system truststore.

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testImportHelpAll.unix.approved.txt

@@ -137,7 +137,8 @@ Truststore:
 
 --tls-hostname-verifier <tls-hostname-verifier>
                      The TLS hostname verification policy for out-going HTTPS and SMTP requests.
-                       Possible values are: ANY, WILDCARD, STRICT. Default: WILDCARD.
+                       Possible values are: ANY, WILDCARD (deprecated), STRICT (deprecated),
+                       DEFAULT. Default: DEFAULT.
 --truststore-paths <truststore-paths>
                      List of pkcs12 (p12 or pfx file extensions), PEM files, or directories
                        containing those files that will be used as a system truststore.

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testStartDevHelp.unix.approved.txt

@@ -256,7 +256,8 @@ Truststore:
 
 --tls-hostname-verifier <tls-hostname-verifier>
                      The TLS hostname verification policy for out-going HTTPS and SMTP requests.
-                       Possible values are: ANY, WILDCARD, STRICT. Default: WILDCARD.
+                       Possible values are: ANY, WILDCARD (deprecated), STRICT (deprecated),
+                       DEFAULT. Default: DEFAULT.
 --truststore-paths <truststore-paths>
                      List of pkcs12 (p12 or pfx file extensions), PEM files, or directories
                        containing those files that will be used as a system truststore.

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testStartDevHelpAll.unix.approved.txt

@@ -304,7 +304,8 @@ Truststore:
 
 --tls-hostname-verifier <tls-hostname-verifier>
                      The TLS hostname verification policy for out-going HTTPS and SMTP requests.
-                       Possible values are: ANY, WILDCARD, STRICT. Default: WILDCARD.
+                       Possible values are: ANY, WILDCARD (deprecated), STRICT (deprecated),
+                       DEFAULT. Default: DEFAULT.
 --truststore-paths <truststore-paths>
                      List of pkcs12 (p12 or pfx file extensions), PEM files, or directories
                        containing those files that will be used as a system truststore.

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testStartHelp.unix.approved.txt

@@ -257,7 +257,8 @@ Truststore:
 
 --tls-hostname-verifier <tls-hostname-verifier>
                      The TLS hostname verification policy for out-going HTTPS and SMTP requests.
-                       Possible values are: ANY, WILDCARD, STRICT. Default: WILDCARD.
+                       Possible values are: ANY, WILDCARD (deprecated), STRICT (deprecated),
+                       DEFAULT. Default: DEFAULT.
 --truststore-paths <truststore-paths>
                      List of pkcs12 (p12 or pfx file extensions), PEM files, or directories
                        containing those files that will be used as a system truststore.

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testStartHelpAll.unix.approved.txt

@@ -305,7 +305,8 @@ Truststore:
 
 --tls-hostname-verifier <tls-hostname-verifier>
                      The TLS hostname verification policy for out-going HTTPS and SMTP requests.
-                       Possible values are: ANY, WILDCARD, STRICT. Default: WILDCARD.
+                       Possible values are: ANY, WILDCARD (deprecated), STRICT (deprecated),
+                       DEFAULT. Default: DEFAULT.
 --truststore-paths <truststore-paths>
                      List of pkcs12 (p12 or pfx file extensions), PEM files, or directories
                        containing those files that will be used as a system truststore.

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testStartOptimizedHelp.unix.approved.txt

@@ -210,7 +210,8 @@ Truststore:
 
 --tls-hostname-verifier <tls-hostname-verifier>
                      The TLS hostname verification policy for out-going HTTPS and SMTP requests.
-                       Possible values are: ANY, WILDCARD, STRICT. Default: WILDCARD.
+                       Possible values are: ANY, WILDCARD (deprecated), STRICT (deprecated),
+                       DEFAULT. Default: DEFAULT.
 --truststore-paths <truststore-paths>
                      List of pkcs12 (p12 or pfx file extensions), PEM files, or directories
                        containing those files that will be used as a system truststore.

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testStartOptimizedHelpAll.unix.approved.txt

@@ -258,7 +258,8 @@ Truststore:
 
 --tls-hostname-verifier <tls-hostname-verifier>
                      The TLS hostname verification policy for out-going HTTPS and SMTP requests.
-                       Possible values are: ANY, WILDCARD, STRICT. Default: WILDCARD.
+                       Possible values are: ANY, WILDCARD (deprecated), STRICT (deprecated),
+                       DEFAULT. Default: DEFAULT.
 --truststore-paths <truststore-paths>
                      List of pkcs12 (p12 or pfx file extensions), PEM files, or directories
                        containing those files that will be used as a system truststore.

services/src/main/java/org/keycloak/connections/httpclient/HttpClientBuilder.java

@@ -20,23 +20,22 @@ package org.keycloak.connections.httpclient;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.conn.ssl.AllowAllHostnameVerifier;
 import org.apache.http.conn.ssl.BrowserCompatHostnameVerifier;
+import org.apache.http.conn.ssl.DefaultHostnameVerifier;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.conn.ssl.SSLContexts;
 import org.apache.http.conn.ssl.StrictHostnameVerifier;
 import org.apache.http.conn.ssl.X509HostnameVerifier;
+import org.apache.http.conn.util.PublicSuffixMatcherLoader;
 import org.apache.http.impl.NoConnectionReuseStrategy;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
+import org.apache.http.ssl.SSLContexts;
 import org.keycloak.common.enums.HostnameVerificationPolicy;
 
 import javax.net.ssl.HostnameVerifier;
 import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLException;
-import javax.net.ssl.SSLSession;
-import javax.net.ssl.SSLSocket;
 import javax.net.ssl.TrustManager;
 import javax.net.ssl.X509TrustManager;
-import java.io.IOException;
 import java.security.KeyManagementException;
 import java.security.KeyStore;
 import java.security.KeyStoreException;
@@ -80,7 +79,7 @@ public class HttpClientBuilder {
     protected KeyStore clientKeyStore;
     protected String clientPrivateKeyPassword;
     protected boolean disableTrustManager;
-    protected HostnameVerificationPolicy policy = HostnameVerificationPolicy.WILDCARD;
+    protected HostnameVerificationPolicy policy = HostnameVerificationPolicy.DEFAULT;
     protected SSLContext sslContext;
     protected int connectionPoolSize = 128;
     protected int maxPooledPerRoute = 64;
@@ -89,7 +88,6 @@ public class HttpClientBuilder {
     protected TimeUnit connectionTTLUnit = TimeUnit.MILLISECONDS;
     protected long maxConnectionIdleTime = 900000;
     protected TimeUnit maxConnectionIdleTimeUnit = TimeUnit.MILLISECONDS;
-    protected HostnameVerifier verifier = null;
     protected long socketTimeout = -1;
     protected TimeUnit socketTimeoutUnits = TimeUnit.MILLISECONDS;
     protected long establishConnectionTimeout = -1;
@@ -215,41 +213,11 @@ public class HttpClientBuilder {
         return this;
     }
 
-    static class VerifierWrapper implements X509HostnameVerifier {
-        protected HostnameVerifier verifier;
-
-        VerifierWrapper(HostnameVerifier verifier) {
-            this.verifier = verifier;
-        }
-
-        @Override
-        public void verify(String host, SSLSocket ssl) throws IOException {
-            if (!verifier.verify(host, ssl.getSession())) throw new SSLException("Hostname verification failure");
-        }
-
-        @Override
-        public void verify(String host, X509Certificate cert) throws SSLException {
-            throw new SSLException("This verification path not implemented");
-        }
-
-        @Override
-        public void verify(String host, String[] cns, String[] subjectAlts) throws SSLException {
-            throw new SSLException("This verification path not implemented");
-        }
-
-        @Override
-        public boolean verify(String s, SSLSession sslSession) {
-            return verifier.verify(s, sslSession);
-        }
-    }
-
     public CloseableHttpClient build() {
-        X509HostnameVerifier verifier = null;
+        HostnameVerifier verifier = null;
-        if (this.verifier != null) verifier = new VerifierWrapper(this.verifier);
-        else {
         switch (policy) {
             case ANY:
-                    verifier = new AllowAllHostnameVerifier();
+                verifier = new NoopHostnameVerifier();
                 break;
             case WILDCARD:
                 verifier = new BrowserCompatHostnameVerifier();
@@ -257,7 +225,9 @@ public class HttpClientBuilder {
             case STRICT:
                 verifier = new StrictHostnameVerifier();
                 break;
-            }
+            case DEFAULT:
+                verifier = new DefaultHostnameVerifier(PublicSuffixMatcherLoader.getDefault());
+                break;
         }
         try {
             SSLConnectionSocketFactory sslsf = null;
@@ -266,7 +236,7 @@ public class HttpClientBuilder {
                 theContext = SSLContext.getInstance("TLS");
                 theContext.init(null, new TrustManager[]{new PassthroughTrustManager()},
                         new SecureRandom());
-                verifier = new AllowAllHostnameVerifier();
+                verifier = new NoopHostnameVerifier();
                 sslsf = new SSLConnectionSocketFactory(theContext, verifier);
             } else if (theContext != null) {
                 sslsf = new SSLConnectionSocketFactory(theContext, verifier);
@@ -324,10 +294,10 @@ public class HttpClientBuilder {
             final SecureRandom random)
             throws NoSuchAlgorithmException, KeyManagementException, KeyStoreException, UnrecoverableKeyException {
         return SSLContexts.custom()
-                        .useProtocol(algorithm)
+                        .setProtocol(algorithm)
                         .setSecureRandom(random)
                         .loadKeyMaterial(keystore, keyPassword != null ? keyPassword.toCharArray() : null)
-                        .loadTrustMaterial(truststore)
+                        .loadTrustMaterial(truststore, null)
                         .build();
     }
 

services/src/main/java/org/keycloak/truststore/FileTruststoreProviderFactory.java

@@ -43,6 +43,8 @@ import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import javax.security.auth.x500.X500Principal;
 
@@ -112,12 +114,15 @@ public class FileTruststoreProviderFactory implements TruststoreProviderFactory
             }
         }
         if (policy == null) {
-            verificationPolicy = HostnameVerificationPolicy.WILDCARD;
+            verificationPolicy = HostnameVerificationPolicy.DEFAULT;
         } else {
             try {
                 verificationPolicy = HostnameVerificationPolicy.valueOf(policy);
             } catch (Exception e) {
-                throw new RuntimeException("Invalid value for 'hostname-verification-policy': " + policy + " (must be one of: ANY, WILDCARD, STRICT)");
+                throw new RuntimeException("Invalid value for 'hostname-verification-policy': " + policy
+                        + " (must be one of: " + Stream.of(HostnameVerificationPolicy.values())
+                                .map(HostnameVerificationPolicy::name).collect(Collectors.joining(", "))
+                        + ")");
             }
         }
 
@@ -158,8 +163,8 @@ public class FileTruststoreProviderFactory implements TruststoreProviderFactory
                 .name(HOSTNAME_VERIFICATION_POLICY)
                 .type("string")
                 .helpText("DEPRECATED: The hostname verification policy.")
-                .options(Arrays.stream(HostnameVerificationPolicy.values()).map(HostnameVerificationPolicy::name).map(String::toLowerCase).toArray(String[]::new))
+                .options(Arrays.stream(HostnameVerificationPolicy.values()).map(HostnameVerificationPolicy::name).toArray(String[]::new))
-                .defaultValue(HostnameVerificationPolicy.WILDCARD.name().toLowerCase())
+                .defaultValue(HostnameVerificationPolicy.DEFAULT.name())
                 .add()
                 .property()
                 .name("type")

services/src/test/java/org/keycloak/truststore/FileTruststoreProviderFactoryTest.java

@@ -36,7 +36,7 @@ public class FileTruststoreProviderFactoryTest {
         factory.init(ScopeUtil.createScope(new HashMap<>()));
         TruststoreProvider provider = factory.create(null);
         assertNotNull(provider.getTruststore());
-        assertEquals(HostnameVerificationPolicy.WILDCARD, provider.getPolicy());
+        assertEquals(HostnameVerificationPolicy.DEFAULT, provider.getPolicy());
     }
 
     @Test

testsuite/integration-arquillian/tests/base/src/test/resources/META-INF/keycloak-server.json

@@ -202,7 +202,7 @@
         "file": {
             "file": "${keycloak.truststore.file:target/dependency/keystore/keycloak.truststore}",
             "password": "${keycloak.truststore.password:secret}",
-            "hostname-verification-policy": "${keycloak.truststore.policy:WILDCARD}",
+            "hostname-verification-policy": "${keycloak.truststore.policy:DEFAULT}",
             "type": "${keycloak.truststore.type:}",
             "disabled": "${keycloak.truststore.disabled:false}"
         }