KEYCLOAK-4743 Add proxy support to HttpClient SPI

We now provide a configurable way for dynamic proxy route selection
for the default HttpClient based on regex based targetHostname patterns.

Introduced `ProxyMapping` to describe a regex based mapping
between target hosts and the proxy URL to use.

A `ProxyMapping` can be build from an ordered list of string based
mapping representations, e.g:
```
   ^.*.(google.com|googleapis.com)$;http://localhost:8080
```
If the targetHost does not match a configured proxy mapping,
no proxy is used.

This can be configured via standalone.xml / jboss-cli, e.g.:
```
   echo SETUP: Configure proxy routes for HttpClient SPI
   /subsystem=keycloak-server/spi=connectionsHttpClient/provider=default:add(enabled=true)
   /subsystem=keycloak-server/spi=connectionsHttpClient/provider=default:write-attribute(name=properties.proxy-mappings,value=["^.*.(google.com|googleapis.com)$;http://www-proxy1:8080","^.*.facebook.com$;http://www-proxy2:8080"])
```
The new `ProxyMappingWareRoutePlanner` uses a configured `ProxyMapping`
to decide which proxy to use for a given request based on the target host
denoted by the HTTP request to execute.

I verified this manually with the BurpProxy Suite.

services/src/main/java/org/keycloak/connections/httpclient/DefaultHttpClientFactory.java

@@ -36,6 +36,8 @@ import org.keycloak.truststore.TruststoreProvider;
 import java.io.IOException;
 import java.io.InputStream;
 import java.security.KeyStore;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -127,6 +129,7 @@ public class DefaultHttpClientFactory implements HttpClientFactory {
                     String clientKeystore = config.get("client-keystore");
                     String clientKeystorePassword = config.get("client-keystore-password");
                     String clientPrivateKeyPassword = config.get("client-key-password");
+                    String[] proxyMappings = config.getArray("proxy-mappings");
 
                     TruststoreProvider truststoreProvider = session.getProvider(TruststoreProvider.class);
                     boolean disableTrustManager = truststoreProvider == null || truststoreProvider.getTruststore() == null;
@@ -137,13 +140,15 @@ public class DefaultHttpClientFactory implements HttpClientFactory {
                             : HttpClientBuilder.HostnameVerificationPolicy.valueOf(truststoreProvider.getPolicy().name());
 
                     HttpClientBuilder builder = new HttpClientBuilder();
+
                     builder.socketTimeout(socketTimeout, TimeUnit.MILLISECONDS)
                             .establishConnectionTimeout(establishConnectionTimeout, TimeUnit.MILLISECONDS)
                             .maxPooledPerRoute(maxPooledPerRoute)
                             .connectionPoolSize(connectionPoolSize)
                             .connectionTTL(connectionTTL, TimeUnit.MILLISECONDS)
                             .maxConnectionIdleTime(maxConnectionIdleTime, TimeUnit.MILLISECONDS)
-                            .disableCookies(disableCookies);
+                            .disableCookies(disableCookies)
+                            .proxyMapping(new ProxyMapping(proxyMappings == null ? Collections.emptyList() : Arrays.asList(proxyMappings)));
 
                     if (disableTrustManager) {
                         // TODO: is it ok to do away with disabling trust manager?

services/src/main/java/org/keycloak/connections/httpclient/HttpClientBuilder.java

@@ -52,7 +52,7 @@ import java.util.concurrent.TimeUnit;
  * @version $Revision: 1 $
  */
 public class HttpClientBuilder {
-    public static enum HostnameVerificationPolicy {
+    public enum HostnameVerificationPolicy {
         /**
          * Hostname verification is not done on the server's certificate
          */
@@ -104,7 +104,7 @@ public class HttpClientBuilder {
     protected long establishConnectionTimeout = -1;
     protected TimeUnit establishConnectionTimeoutUnits = TimeUnit.MILLISECONDS;
     protected boolean disableCookies = false;
-
+    protected ProxyMapping proxyMapping;
 
     /**
      * Socket inactivity timeout
@@ -208,6 +208,11 @@ public class HttpClientBuilder {
         return this;
     }
 
+    public HttpClientBuilder proxyMapping(ProxyMapping proxyMapping) {
+        this.proxyMapping = proxyMapping;
+        return this;
+    }
+
 
     static class VerifierWrapper implements X509HostnameVerifier {
         protected HostnameVerifier verifier;
@@ -272,6 +277,7 @@ public class HttpClientBuilder {
                 tlsContext.init(null, null, null);
                 sslsf = new SSLConnectionSocketFactory(tlsContext, verifier);
             }
+
             RequestConfig requestConfig = RequestConfig.custom()
                     .setConnectTimeout((int) establishConnectionTimeout)
                     .setSocketTimeout((int) socketTimeout).build();
@@ -283,6 +289,11 @@ public class HttpClientBuilder {
                     .setMaxConnPerRoute(maxPooledPerRoute)
                     .setConnectionTimeToLive(connectionTTL, connectionTTLUnit);
 
+
+            if (proxyMapping != null && !proxyMapping.isEmpty()) {
+                builder.setRoutePlanner(new ProxyMappingAwareRoutePlanner(proxyMapping));
+            }
+
             if (maxConnectionIdleTime > 0) {
                 // Will start background cleaner thread
                 builder.evictIdleConnections(maxConnectionIdleTime, maxConnectionIdleTimeUnit);

services/src/main/java/org/keycloak/connections/httpclient/ProxyMapping.java

@@ -0,0 +1,113 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.connections.httpclient;
+
+import org.apache.http.HttpHost;
+
+import java.net.URI;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.regex.Pattern;
+
+/**
+ * {@link ProxyMapping} describes mapping for hostname regex patterns to a {@link HttpHost} proxy.
+ *
+ * @author <a href="mailto:thomas.darimont@gmail.com">Thomas Darimont</a>
+ */
+public class ProxyMapping {
+
+  private static final String DELIMITER = ";";
+
+  private final Map<Pattern, HttpHost> hostPatternToProxyHost;
+
+  /**
+   * Creates a new  {@link ProxyMapping} from the provided {@code List} of proxy mapping strings.
+   * <p>
+   * A proxy mapping string must have the following format: {@code hostnameRegex;www-proxy-uri } with semicolon as a delimiter.
+   * This format enables easy configuration via SPI config string in standalone.xml.
+   * </p>
+   * <p>For example
+   * {@code ^.*.(google.com|googleapis.com)$;http://www-proxy.mycorp.local:8080}
+   * </p>
+   *
+   * @param mappings
+   */
+  public ProxyMapping(List<String> mappings) {
+    this(parseProxyMappings(mappings));
+  }
+
+  /**
+   * Creates a {@link ProxyMapping} from the provided mappings.
+   *
+   * @param mappings
+   */
+  public ProxyMapping(Map<Pattern, HttpHost> mappings) {
+    this.hostPatternToProxyHost = Collections.unmodifiableMap(mappings);
+  }
+
+  private static Map<Pattern, HttpHost> parseProxyMappings(List<String> mapping) {
+
+    if (mapping == null || mapping.isEmpty()) {
+      return Collections.emptyMap();
+    }
+
+    // Preserve the order provided via mapping
+    Map<Pattern, HttpHost> map = new LinkedHashMap<>();
+
+    for (String entry : mapping) {
+      String[] hostPatternRegexWithProxyHost = entry.split(DELIMITER);
+      String hostPatternRegex = hostPatternRegexWithProxyHost[0];
+      String proxyUrl = hostPatternRegexWithProxyHost[1];
+
+      URI uri = URI.create(proxyUrl);
+      HttpHost proxy = new HttpHost(uri.getHost(), uri.getPort(), uri.getScheme());
+
+      Pattern hostPattern = Pattern.compile(hostPatternRegex);
+      map.put(hostPattern, proxy);
+    }
+
+    return map;
+  }
+
+  public boolean isEmpty() {
+    return this.hostPatternToProxyHost.isEmpty();
+  }
+
+  /**
+   * @param hostname
+   * @return the {@link HttpHost} proxy associated with the first matching hostname {@link Pattern} or {@literal null} if none matches.
+   */
+  public HttpHost getProxyFor(String hostname) {
+
+    Objects.requireNonNull(hostname, "hostname");
+
+    for (Map.Entry<Pattern, HttpHost> entry : hostPatternToProxyHost.entrySet()) {
+
+      Pattern hostnamePattern = entry.getKey();
+      HttpHost proxy = entry.getValue();
+
+      if (hostnamePattern.matcher(hostname).matches()) {
+        return proxy;
+      }
+    }
+
+    return null;
+  }
+}

services/src/main/java/org/keycloak/connections/httpclient/ProxyMappingAwareRoutePlanner.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.connections.httpclient;
+
+import org.apache.http.HttpException;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpRequest;
+import org.apache.http.impl.conn.DefaultRoutePlanner;
+import org.apache.http.impl.conn.DefaultSchemePortResolver;
+import org.apache.http.protocol.HttpContext;
+import org.jboss.logging.Logger;
+
+/**
+ * A {@link DefaultRoutePlanner} that determines the proxy to use for a given target hostname by consulting a {@link ProxyMapping}.
+ *
+ * @author <a href="mailto:thomas.darimont@gmail.com">Thomas Darimont</a>
+ */
+public class ProxyMappingAwareRoutePlanner extends DefaultRoutePlanner {
+
+  private static final Logger LOG = Logger.getLogger(ProxyMappingAwareRoutePlanner.class);
+
+  private final ProxyMapping proxyMapping;
+
+  public ProxyMappingAwareRoutePlanner(ProxyMapping proxyMapping) {
+    super(DefaultSchemePortResolver.INSTANCE);
+    this.proxyMapping = proxyMapping;
+  }
+
+  @Override
+  protected HttpHost determineProxy(HttpHost target, HttpRequest request, HttpContext context) throws HttpException {
+
+    HttpHost proxy = proxyMapping.getProxyFor(target.getHostName());
+
+    LOG.debugf("Returning proxy=%s for targetHost=%s", proxy ,target.getHostName());
+
+    return proxy;
+  }
+}

services/src/test/java/org/keycloak/connections/httpclient/ProxyMappingTest.java

@@ -0,0 +1,104 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.connections.httpclient;
+
+import org.apache.http.HttpHost;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.hamcrest.CoreMatchers.nullValue;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author <a href="mailto:thomas.darimont@gmail.com">Thomas Darimont</a>
+ */
+public class ProxyMappingTest {
+
+  private static final List<String> DEFAULT_MAPPINGS = Arrays.asList( //
+    "^.*.(google.com|googleapis.com)$;http://proxy1:8080", //
+    "^.*.(facebook.com)$;http://proxy2:8080" //
+  );
+
+  @Rule
+  public ExpectedException expectedException = ExpectedException.none();
+
+  ProxyMapping proxyMapping;
+
+  @Before
+  public void setup() {
+    proxyMapping = new ProxyMapping(DEFAULT_MAPPINGS);
+  }
+
+  @Test
+  public void proxyMappingFromEmptyMapShouldBeEmpty() {
+    assertThat(new ProxyMapping(Collections.emptyMap()).isEmpty(), is(true));
+  }
+
+  @Test
+  public void proxyMappingFromEmptyListShouldBeEmpty() {
+    assertThat(new ProxyMapping(new ArrayList<>()).isEmpty(), is(true));
+  }
+
+  @Test
+  public void shouldReturnProxy1ForConfiguredProxyMapping() {
+
+    HttpHost proxy = proxyMapping.getProxyFor("account.google.com");
+    assertThat(proxy, is(notNullValue()));
+    assertThat(proxy.getHostName(), is("proxy1"));
+  }
+
+  @Test
+  public void shouldReturnProxy1ForConfiguredProxyMappingWithSubDomain() {
+
+    HttpHost proxy = proxyMapping.getProxyFor("awesome.account.google.com");
+    assertThat(proxy, is(notNullValue()));
+    assertThat(proxy.getHostName(), is("proxy1"));
+  }
+
+  @Test
+  public void shouldReturnProxy2ForConfiguredProxyMapping() {
+
+    HttpHost proxy = proxyMapping.getProxyFor("login.facebook.com");
+    assertThat(proxy, is(notNullValue()));
+    assertThat(proxy.getHostName(), is("proxy2"));
+  }
+
+  @Test
+  public void shouldReturnNoProxyForUnknownHost() {
+
+    HttpHost proxy = proxyMapping.getProxyFor("login.microsoft.com");
+    assertThat(proxy, is(nullValue()));
+  }
+
+  @Test
+  public void shouldRejectNull() {
+
+    expectedException.expect(NullPointerException.class);
+    expectedException.expectMessage("hostname");
+
+    proxyMapping.getProxyFor(null);
+  }
+}