[KEYCLOAK-10935] Add a vault transcriber implementation that can be obtained from the session.

- automatically parses ${vault.<KEY>} expressions to obtain the key that contains the secret in the vault.
 - enchances the capabilities of the VaultProvider by offering methods to convert the raw secrets into other types.

server-spi-private/src/main/java/org/keycloak/vault/VaultProvider.java

@@ -28,7 +28,7 @@ public interface VaultProvider extends Provider {
      * Retrieves a secret from vault. The implementation should respect
      * at least the realm ID to separate the secrets within the vault.
      * If the secret is retrieved successfully, it is returned;
-     * otherwise this method results into an empty {@link VaultRawSecret#getRawSecret()}.
+     * otherwise this method results into an empty {@link VaultRawSecret#get()}.
      *
      * This method is intended to be used within a try-with-resources block so that
      * the secret is destroyed immediately after use.
@@ -42,7 +42,7 @@ public interface VaultProvider extends Provider {
      *
      * @return Always a non-{@code null} value with the raw secret.
      *         Within the returned value, the secret or {@code null} is stored in the
-     *         {@link VaultRawSecret#getRawSecret()} return value if the secret was successfully
+     *         {@link VaultRawSecret#get()} return value if the secret was successfully
      *         resolved, or an empty {@link java.util.Optional} if the secret has not been found in the vault.
      */
     VaultRawSecret obtainSecret(String vaultSecretId);

server-spi/src/main/java/org/keycloak/models/KeycloakSession.java

@@ -22,6 +22,7 @@ import org.keycloak.models.cache.UserCache;
 import org.keycloak.provider.Provider;
 import org.keycloak.sessions.AuthenticationSessionProvider;
 import org.keycloak.storage.federated.UserFederatedStorageProvider;
+import org.keycloak.vault.VaultTranscriber;
 
 import java.util.Set;
 
@@ -196,4 +197,8 @@ public interface KeycloakSession {
      */
     TokenManager tokens();
 
+    /**
+     * Vault transcriber
+     */
+    VaultTranscriber vault();
 }

server-spi/src/main/java/org/keycloak/vault/VaultCharSecret.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright 2019 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.vault;
+
+import java.nio.CharBuffer;
+import java.util.Optional;
+
+/**
+ * A {@link CharBuffer} based representation of the secret obtained from the vault that supports automated cleanup of memory.
+ *
+ * @author <a href="mailto:sguilhen@redhat.com">Stefan Guilhen</a>
+ */
+public interface VaultCharSecret extends AutoCloseable {
+
+    /**
+     * Returns the secret enclosed in a {@link CharBuffer}.
+     *
+     * @return If the secret was successfully resolved by vault, returns an {@link Optional} containing the value returned
+     *         by the vault as a {@link CharBuffer} (a valid value can be {@code null}), or an empty {@link Optional}
+     */
+    Optional<CharBuffer> get();
+
+    /**
+     * Returns the secret in its {@code char[]} form.
+     *
+     * @return If the secret was successfully resolved by vault, returns an {@link Optional} containing the value returned
+     *         by the vault as a {@code char[]} (a valid value can be {@code null}), or an empty {@link Optional}.
+     */
+    Optional<char[]> getAsArray();
+
+    /**
+     *  Destroys the secret in memory by e.g. overwriting it with random garbage.
+     */
+    @Override
+    void close();
+}

server-spi/src/main/java/org/keycloak/vault/VaultRawSecret.java

@@ -32,7 +32,15 @@ public interface VaultRawSecret extends AutoCloseable {
      *         an {@link Optional} containing the value returned by the vault
      *         (a valid value can be {@code null}), or an empty {@link Optional}
      */
-    Optional<ByteBuffer> getRawSecret();
+    Optional<ByteBuffer> get();
+
+    /**
+     * Returns the raw secret bytes in {@code byte[]} form.
+     * @return If the secret was successfully resolved by vault, returns
+     *         an {@link Optional} containing the value returned by the vault
+     *         (a valid value can be {@code null}), or an empty {@link Optional}
+     */
+    Optional<byte[]> getAsArray();
 
     /**
      *  Destroys the secret in memory by e.g. overwriting it with random garbage.

server-spi/src/main/java/org/keycloak/vault/VaultStringSecret.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2019 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.vault;
+
+import java.util.Optional;
+
+/**
+ * A {@link String} based representation of the secret obtained from the vault that supports automated cleanup of memory.
+ * In this case, due to the immutable nature of strings, the cleanup should consist in releasing any references to the
+ * secret string so it can be disposed by the GC as soon as possible.
+ *
+ * @author <a href="mailto:sguilhen@redhat.com">Stefan Guilhen</a>
+ */
+public interface VaultStringSecret extends AutoCloseable {
+
+    /**
+     * Returns the secret represented as a {@link String}.
+     * @return If the secret was successfully resolved by vault, returns an {@link Optional} containing the value returned
+     *         by the vault as a {@link String} (a valid value can be {@code null}), or an empty {@link Optional}
+     */
+    Optional<String> get();
+
+    /**
+     *  Destroys the secret in memory by e.g. overwriting it with random garbage or release references in case of immutable
+     *  secrets.
+     */
+    @Override
+    void close();
+}

server-spi/src/main/java/org/keycloak/vault/VaultTranscriber.java

@@ -0,0 +1,88 @@
+/*
+ * Copyright 2019 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.vault;
+
+import java.lang.ref.WeakReference;
+import java.nio.CharBuffer;
+import java.util.Optional;
+
+/**
+ * A facade to the configured vault provider that exposes utility methods for obtaining the vault secrets in different
+ * formats (such as {@link VaultRawSecret}, {@link VaultCharSecret} or {@link VaultStringSecret}).
+ *
+ * @see VaultRawSecret
+ * @see VaultCharSecret
+ *
+ * @author <a href="mailto:sguilhen@redhat.com">Stefan Guilhen</a>
+ */
+public interface VaultTranscriber {
+
+    /**
+     * Obtains the raw secret from the vault that matches the entry in the specified value string. The value must follow
+     * the format {@code ${vault.<KEY>}} where {@code <KEY>} identifies the entry in the vault. If the value doesn't follow
+     * the vault expression format, it is assumed to be the secret itself and is encoded into a {@link VaultRawSecret}.
+     * <p/>
+     * The returned {@link VaultRawSecret} extends {@link AutoCloseable} and it is strongly recommended that it is used in
+     * try-with-resources blocks to ensure the raw secret is overridden (destroyed) when the calling code is finished using
+     * it.
+     *
+     * @param value a {@link String} that might be a vault expression containing a vault entry key.
+     * @return a {@link VaultRawSecret} representing the secret that was read from the vault. If the specified value is not
+     *         a vault expression then the returned secret is the value itself encoded as a {@link VaultRawSecret}.
+     */
+    VaultRawSecret getRawSecret(final String value);
+
+    /**
+     * Obtains the secret represented as a {@link VaultCharSecret} from the vault that matches the entry in the specified
+     * value string. The value must follow the format {@code ${vault.<KEY>}} where {@code <KEY>} identifies the entry in
+     * the vault. If the value doesn't follow the vault expression format, it is assumed to be the secret itself and is
+     * encoded into a {@link VaultCharSecret}.
+     * <p/>
+     * The returned {@link VaultCharSecret} extends {@link AutoCloseable} and it is strongly recommended that it is used in
+     * try-with-resources blocks to ensure the raw secret is overridden (destroyed) when the calling code is finished using
+     * it.
+     *
+     * @param value a {@link String} that might be a vault expression containing a vault entry key.
+     * @return a {@link VaultRawSecret} representing the secret that was read from the vault. If the specified value is not
+     *         a vault expression then the returned secret is the value itself encoded as a {@link VaultRawSecret}.
+     */
+    VaultCharSecret getCharSecret(final String value);
+
+    /**
+     * Obtains the secret represented as a {@link String} from the vault that matches the entry in the specified value.
+     * The value must follow the format {@code ${vault.<KEY>}} where {@code <KEY>} identifies the entry in the vault. If
+     * the value doesn't follow the vault expression format, it is assumed to be the secret itself.
+     * <p/>
+     * Due to the immutable nature of strings and the way the JVM handles them internally, implementations that keep a reference
+     * to the secret string might consider doing so using a {@link WeakReference} that can be cleared in the {@link AutoCloseable#close()}
+     * method. Being immutable, such strings cannot be overridden (destroyed) by the implementation, but using a {@link WeakReference}
+     * guarantees that at least no hard references to the secret are held by the implementation class itself (which would
+     * prevent proper GC disposal of the secrets).
+     * <p/>
+     * <b>WARNING:</b> It is strongly recommended that callers of this method use the returned secret in try-with-resources
+     * blocks and they should strive not to keep hard references to the enclosed secret string for any longer than necessary
+     * so that the secret becomes available for GC as soon as possible. These measures help shorten the window of time when
+     * the secret strings are readable from memory.
+     *
+     * @param value a {@link String} that might be a vault expression containing a vault entry key.
+     * @return a {@link VaultStringSecret} representing the secret that was read from the vault. If the specified value is not
+     *         a vault expression then the returned secret is the value itself encoded as a {@link VaultStringSecret}.
+     */
+    VaultStringSecret getStringSecret(final String value);
+
+}

services/src/main/java/org/keycloak/services/DefaultKeycloakSession.java

@@ -42,6 +42,9 @@ import org.keycloak.storage.ClientStorageManager;
 import org.keycloak.storage.UserStorageManager;
 import org.keycloak.storage.federated.UserFederatedStorageProvider;
 import org.keycloak.theme.DefaultThemeManager;
+import org.keycloak.vault.DefaultVaultTranscriber;
+import org.keycloak.vault.VaultProvider;
+import org.keycloak.vault.VaultTranscriber;
 
 import java.util.HashMap;
 import java.util.HashSet;
@@ -71,6 +74,7 @@ public class DefaultKeycloakSession implements KeycloakSession {
     private KeyManager keyManager;
     private ThemeManager themeManager;
     private TokenManager tokenManager;
+    private VaultTranscriber vaultTranscriber;
 
     public DefaultKeycloakSession(DefaultKeycloakSessionFactory factory) {
         this.factory = factory;
@@ -302,6 +306,14 @@ public class DefaultKeycloakSession implements KeycloakSession {
         return tokenManager;
     }
 
+    @Override
+    public VaultTranscriber vault() {
+        if (this.vaultTranscriber == null) {
+            this.vaultTranscriber = new DefaultVaultTranscriber(this.getProvider(VaultProvider.class));
+        }
+        return this.vaultTranscriber;
+    }
+
     public void close() {
         for (Provider p : providers.values()) {
             try {

services/src/main/java/org/keycloak/vault/DefaultVaultCharSecret.java

@@ -0,0 +1,95 @@
+/*
+ * Copyright 2019 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.vault;
+
+import java.nio.CharBuffer;
+import java.util.Optional;
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ * Default {@link VaultCharSecret} implementation based on {@link CharBuffer}.
+ *
+ * @author <a href="mailto:sguilhen@redhat.com">Stefan Guilhen</a>
+ */
+public class DefaultVaultCharSecret implements VaultCharSecret {
+
+    private static final VaultCharSecret EMPTY_VAULT_SECRET = new VaultCharSecret() {
+        @Override
+        public Optional<CharBuffer> get() {
+            return Optional.empty();
+        }
+
+        @Override
+        public Optional<char[]> getAsArray() {
+            return Optional.empty();
+        }
+
+        @Override
+        public void close() {
+        }
+    };
+
+    public static VaultCharSecret forBuffer(Optional<CharBuffer> buffer) {
+        if (buffer == null || ! buffer.isPresent()) {
+            return EMPTY_VAULT_SECRET;
+        }
+        return new DefaultVaultCharSecret(buffer.get());
+    }
+
+    private final CharBuffer buffer;
+
+    private char[] secretArray;
+
+    private DefaultVaultCharSecret(final CharBuffer buffer) {
+        this.buffer = buffer;
+    }
+
+    @Override
+    public Optional<CharBuffer> get() {
+        return Optional.of(this.buffer);
+    }
+
+    @Override
+    public Optional<char[]> getAsArray() {
+        if (this.secretArray == null) {
+            // initialize internal array on demand.
+            if (this.buffer.hasArray()) {
+                this.secretArray = buffer.array();
+            } else {
+                secretArray = new char[buffer.capacity()];
+                buffer.get(secretArray);
+            }
+        }
+        return Optional.of(this.secretArray);
+    }
+
+    @Override
+    public void close() {
+        if (this.buffer.hasArray()) {
+            char[] internalArray = this.buffer.array();
+            for (int i = 0; i < internalArray.length; i++) {
+                internalArray[i] = (char) ThreadLocalRandom.current().nextInt();
+            }
+        } else if (this.secretArray != null) {
+            for (int i = 0; i < this.secretArray.length; i++) {
+                this.secretArray[i] = (char) ThreadLocalRandom.current().nextInt();
+            }
+        }
+        this.buffer.clear();
+    }
+}

services/src/main/java/org/keycloak/vault/DefaultVaultRawSecret.java

@@ -28,7 +28,12 @@ public class DefaultVaultRawSecret implements VaultRawSecret {
 
     private static final VaultRawSecret EMPTY_VAULT_SECRET = new VaultRawSecret() {
         @Override
-        public Optional<ByteBuffer> getRawSecret() {
+        public Optional<ByteBuffer> get() {
+            return Optional.empty();
+        }
+
+        @Override
+        public Optional<byte[]> getAsArray() {
             return Optional.empty();
         }
 
@@ -39,6 +44,8 @@ public class DefaultVaultRawSecret implements VaultRawSecret {
 
     private final ByteBuffer rawSecret;
 
+    private byte[] secretArray;
+
     public static VaultRawSecret forBuffer(Optional<ByteBuffer> buffer) {
         if (buffer == null || ! buffer.isPresent()) {
             return EMPTY_VAULT_SECRET;
@@ -51,14 +58,30 @@ public class DefaultVaultRawSecret implements VaultRawSecret {
     }
 
     @Override
-    public Optional<ByteBuffer> getRawSecret() {
+    public Optional<ByteBuffer> get() {
         return Optional.of(this.rawSecret);
     }
 
+    @Override
+    public Optional<byte[]> getAsArray() {
+        if (this.secretArray == null) {
+            // initialize internal array on demand.
+            if (this.rawSecret.hasArray()) {
+                this.secretArray = this.rawSecret.array();
+            } else {
+                secretArray = new byte[this.rawSecret.capacity()];
+                this.rawSecret.get(secretArray);
+            }
+        }
+        return Optional.of(this.secretArray);
+    }
+
     @Override
     public void close() {
         if (rawSecret.hasArray()) {
             ThreadLocalRandom.current().nextBytes(rawSecret.array());
+        } else if (this.secretArray != null) {
+            ThreadLocalRandom.current().nextBytes(this.secretArray);
         }
         rawSecret.clear();
     }

services/src/main/java/org/keycloak/vault/DefaultVaultStringSecret.java

@@ -0,0 +1,46 @@
+package org.keycloak.vault;
+
+import java.lang.ref.WeakReference;
+import java.util.Optional;
+
+/**
+ * Default {@link VaultCharSecret} implementation based on {@link String}.
+ *
+ * @author <a href="mailto:sguilhen@redhat.com">Stefan Guilhen</a>
+ */
+public class DefaultVaultStringSecret implements VaultStringSecret {
+
+    private static final VaultStringSecret EMPTY_VAULT_SECRET = new VaultStringSecret() {
+        @Override
+        public Optional<String> get() {
+            return Optional.empty();
+        }
+
+        @Override
+        public void close() {
+        }
+    };
+
+    public static VaultStringSecret forString(Optional<String> secret) {
+        if (secret == null || ! secret.isPresent()) {
+            return EMPTY_VAULT_SECRET;
+        }
+        return new DefaultVaultStringSecret(secret.get());
+    }
+
+    private String secret;
+
+    private DefaultVaultStringSecret(final String secret) {
+        this.secret = secret;
+    }
+
+    @Override
+    public Optional<String> get() {
+        return Optional.of(this.secret);
+    }
+
+    @Override
+    public void close() {
+        this.secret = null;
+    }
+}

services/src/main/java/org/keycloak/vault/DefaultVaultTranscriber.java

@@ -0,0 +1,116 @@
+/*
+ * Copyright 2019 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.vault;
+
+import java.lang.ref.WeakReference;
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Default {@link VaultTranscriber} implementation that uses the configured {@link VaultProvider} to obtain raw secrets
+ * and convert them into other types. By default, the {@link VaultProvider} provides raw secrets through a {@link ByteBuffer}.
+ * This class offers methods to convert the raw secrets into other types (such as {@link VaultCharSecret} or {@link WeakReference<String>}).
+ *
+ * @see VaultRawSecret
+ * @see VaultCharSecret
+ *
+ * @author <a href="mailto:sguilhen@redhat.com">Stefan Guilhen</a>
+ */
+public class DefaultVaultTranscriber implements VaultTranscriber {
+
+    private static final Pattern pattern = Pattern.compile("^\\$\\{vault\\.(.+?)}$");
+
+    private final VaultProvider provider;
+
+    public DefaultVaultTranscriber(final VaultProvider provider) {
+        if (provider == null) {
+            this.provider = new VaultProvider() {
+                @Override
+                public VaultRawSecret obtainSecret(String vaultSecretId) {
+                    return DefaultVaultRawSecret.forBuffer(null);
+                }
+
+                @Override
+                public void close() {
+                }
+            };
+        } else {
+            this.provider = provider;
+        }
+    }
+
+    @Override
+    public VaultRawSecret getRawSecret(final String value) {
+        String entryId = this.getVaultEntryKey(value);
+        if (entryId != null) {
+            // we have a valid ${vault.<KEY>} string, use the provider to retrieve the entry.
+            return this.provider.obtainSecret(entryId);
+        } else {
+            // not a vault expression - encode the value itself as a byte buffer.
+            ByteBuffer buffer = value != null ? ByteBuffer.wrap(value.getBytes(StandardCharsets.UTF_8)) : null;
+            return DefaultVaultRawSecret.forBuffer(Optional.ofNullable(buffer));
+        }
+    }
+
+    @Override
+    public VaultCharSecret getCharSecret(final String value) {
+        // obtain the raw secret and convert it into a char secret.
+        try (VaultRawSecret rawSecret = this.getRawSecret(value)) {
+            if (!rawSecret.get().isPresent()) {
+                return DefaultVaultCharSecret.forBuffer(Optional.empty());
+            }
+            ByteBuffer rawSecretBuffer = rawSecret.get().get();
+            CharBuffer charSecretBuffer = StandardCharsets.UTF_8.decode(rawSecretBuffer);
+            return DefaultVaultCharSecret.forBuffer(Optional.of(charSecretBuffer));
+        }
+    }
+
+    @Override
+    public VaultStringSecret getStringSecret(final String value) {
+        // obtain the raw secret and convert it into a string string.
+        try (VaultRawSecret rawSecret = this.getRawSecret(value)) {
+            if (!rawSecret.get().isPresent()) {
+                return DefaultVaultStringSecret.forString(Optional.empty());
+            }
+            ByteBuffer rawSecretBuffer = rawSecret.get().get();
+            return DefaultVaultStringSecret.forString(Optional.of(StandardCharsets.UTF_8.decode(rawSecretBuffer).toString()));
+        }
+    }
+
+    /**
+     * Obtains the vault entry key from the specified value if the value is a valid {@code ${vault.<KEY>}} expression.
+     * For example, calling this method with the {@code ${vault.smtp_secret}} argument results in the string {@code smtp_secret}
+     * being returned.
+     *
+     * @param value a {@code String} that might contain a vault entry key.
+     * @return the extracted entry key if the value follows the {@code ${vault.<KEY>}} format; null otherwise.
+     */
+    private String getVaultEntryKey(final String value) {
+        if (value != null) {
+            Matcher matcher = pattern.matcher(value);
+            if (matcher.matches()) {
+                return matcher.group(1);
+            }
+        }
+        return null;
+    }
+}

services/src/test/java/org/keycloak/vault/PlainTextVaultProviderTest.java

@@ -29,7 +29,7 @@ public class PlainTextVaultProviderTest {
 
         //then
         assertNotNull(secret1);
-        assertNotNull(secret1.getRawSecret().get());
+        assertNotNull(secret1.get().get());
         assertThat(secret1, secretContains("secret1"));
     }
 
@@ -43,7 +43,7 @@ public class PlainTextVaultProviderTest {
 
         //then
         assertNotNull(secret1);
-        assertNotNull(secret1.getRawSecret().get());
+        assertNotNull(secret1.get().get());
         assertThat(secret1, secretContains("underscore_secret1"));
     }
 
@@ -57,7 +57,7 @@ public class PlainTextVaultProviderTest {
 
         //then
         assertNotNull(secret);
-        assertFalse(secret.getRawSecret().isPresent());
+        assertFalse(secret.get().isPresent());
     }
 
     @Test
@@ -70,7 +70,7 @@ public class PlainTextVaultProviderTest {
 
         //then
         assertNotNull(secret);
-        assertFalse(secret.getRawSecret().isPresent());
+        assertFalse(secret.get().isPresent());
     }
 
     @Test
@@ -93,12 +93,12 @@ public class PlainTextVaultProviderTest {
 
         Files.write(temporarySecretFile, "secret1".getBytes());
         try (VaultRawSecret secret1 = provider.obtainSecret(secretName)) {
-            secret1AsString = StandardCharsets.UTF_8.decode(secret1.getRawSecret().get()).toString();
+            secret1AsString = StandardCharsets.UTF_8.decode(secret1.get().get()).toString();
         }
 
         Files.write(temporarySecretFile, "secret2".getBytes());
         try (VaultRawSecret secret2 = provider.obtainSecret(secretName)) {
-            secret2AsString = StandardCharsets.UTF_8.decode(secret2.getRawSecret().get()).toString();
+            secret2AsString = StandardCharsets.UTF_8.decode(secret2.get().get()).toString();
         }
 
         //then

services/src/test/java/org/keycloak/vault/SecretContains.java

@@ -19,7 +19,7 @@ public class SecretContains extends TypeSafeMatcher<VaultRawSecret> {
 
     @Override
     protected boolean matchesSafely(VaultRawSecret secret) {
-        String convertedSecret = StandardCharsets.UTF_8.decode(secret.getRawSecret().get()).toString();
+        String convertedSecret = StandardCharsets.UTF_8.decode(secret.get().get()).toString();
         return thisVaultAsString.equals(convertedSecret);
     }
 

services/src/test/java/org/keycloak/vault/VaultTranscriberTest.java

@@ -0,0 +1,361 @@
+/*
+ * Copyright 2019 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.vault;
+
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Tests for the {@link DefaultVaultTranscriber} implementation.
+ *
+ * @author <a href="mailto:sguilhen@redhat.com">Stefan Guilhen</a>
+ */
+public class VaultTranscriberTest {
+
+    private final VaultTranscriber transcriber = new DefaultVaultTranscriber(new TestVaultProvider());
+
+    private static Map<String, String> validExpressions;
+
+    private static String[] invalidExpressions;
+
+    @BeforeClass
+    public static void init() {
+        validExpressions = new HashMap<>();
+        // expressions with keys that exist in the vault.
+        validExpressions.put("${vault.vault_key_1}", "secret1");
+        validExpressions.put("${vault.vault_key_2}", "secret2");
+        // expressions with keys that don't exist in the vault.
+        validExpressions.put("${vault.invalid_key}", null);
+        validExpressions.put("${vault.${.id-!@#$%^&*_()}}", null);
+        // invalid expressions.
+        invalidExpressions = new String[]{"${vault.}","$vault.id}", "{vault.id}", "${vault.id", "${vaultid}", ""};
+
+    }
+
+    /**
+     * Tests the retrieval of raw secrets using valid vault expressions - i.e. expressions that identify the key that should be
+     * used to retrieve the secret from the vault.
+     * <p/>
+     * Some of the keys used in this test exist in the test vault while others, despite being valid expressions, don't identify
+     * any secret in the vault. For the former, the test compares the obtained secret against the expected secret (using both
+     * the buffer and array representations of the secret) and then checks if the secrets have been overridden/destroyed after
+     * the try-wih-resources block. For the latter, the tests checks if an empty {@link Optional} has been returned by the
+     * transcriber.
+     *
+     * @throws Exception if an error occurs while running the test.
+     */
+    @Test
+    public void testGetRawSecretUsingValidExpressions() throws Exception {
+        ByteBuffer secretBuffer = null;
+        byte[] secretArray = null;
+
+        // attempt to obtain a secret using a proper vault expressions. The key may or may not exist in the vault, so we
+        // check both cases using the returned optional and comparing against the expected secret.
+        for (String key : validExpressions.keySet()) {
+            String expectedSecret = validExpressions.get(key);
+            try (VaultRawSecret secret = transcriber.getRawSecret(key)) {
+                Optional<ByteBuffer> optional = secret.get();
+                Optional<byte[]> optionalArray = secret.getAsArray();
+                if (expectedSecret != null) {
+                    Assert.assertTrue(optional.isPresent());
+                    secretBuffer = optional.get();
+                    Assert.assertArrayEquals(expectedSecret.getBytes(StandardCharsets.UTF_8), secretBuffer.array());
+                    Assert.assertTrue(optionalArray.isPresent());
+                    secretArray = optionalArray.get();
+                    Assert.assertArrayEquals(expectedSecret.getBytes(StandardCharsets.UTF_8), secretArray);
+                } else {
+                    Assert.assertFalse(optional.isPresent());
+                    Assert.assertFalse(optionalArray.isPresent());
+                }
+            }
+            // after the try-with-resources block the secret should have been overridden.
+            if (expectedSecret != null) {
+                Assert.assertFalse(Arrays.equals(expectedSecret.getBytes(StandardCharsets.UTF_8), secretBuffer.array()));
+                Assert.assertFalse(Arrays.equals(expectedSecret.getBytes(StandardCharsets.UTF_8), secretArray));
+            }
+        }
+    }
+
+    /**
+     * Tests the retrieval of raw secrets using invalid vault expressions - i.e. expressions that identify the key that should be
+     * used to retrieve the secret from the vault. When the values supplied to the transcriber are not valid vault expressions
+     * the value itself is assumed to be the secret and is enclosed in the secret class that is returned. Thus this test
+     * checks if the returned secret matches the specified values (using both the buffer and array representation of the value)
+     * and then checks if the secrets have been overridden/destroyed after the try-wih-resources block.
+     *
+     * @throws Exception if an error occurs while running the test.
+     */
+    @Test
+    public void testGetRawSecretUsingInvalidExpressions() throws Exception {
+        ByteBuffer secretBuffer;
+        byte[] secretArray;
+
+        // attempt to obtain a secret using invalid vault expressions - the value itself should be returned as a byte buffer.
+        for (String value : this.invalidExpressions) {
+            try (VaultRawSecret secret = transcriber.getRawSecret(value)) {
+                Optional<ByteBuffer> optional = secret.get();
+                Optional<byte[]> optionalArray = secret.getAsArray();
+                Assert.assertTrue(optional.isPresent());
+                secretBuffer = optional.get();
+                Assert.assertArrayEquals(value.getBytes(StandardCharsets.UTF_8), secretBuffer.array());
+                Assert.assertTrue(optionalArray.isPresent());
+                secretArray = optionalArray.get();
+                Assert.assertArrayEquals(value.getBytes(StandardCharsets.UTF_8), secretArray);
+            }
+            // after the try-with-resources block the secret should have been overridden.
+            if (!value.isEmpty()) {
+                Assert.assertFalse(Arrays.equals(value.getBytes(StandardCharsets.UTF_8), secretBuffer.array()));
+                Assert.assertFalse(Arrays.equals(value.getBytes(StandardCharsets.UTF_8), secretArray));
+            }
+        }
+    }
+
+    /**
+     * Tests that a null vault expression always returns an empty secret.
+     *
+     * @throws Exception if an error occurs while running the test.
+     */
+    @Test
+    public void testGetRawSecretUsingNullExpression() throws Exception {
+        // check that a null expression results in an empty optional instance.
+        try (VaultRawSecret secret = transcriber.getRawSecret(null)) {
+            Assert.assertFalse(secret.get().isPresent());
+            Assert.assertFalse(secret.getAsArray().isPresent());
+        }
+    }
+
+    /**
+     * Tests the retrieval of char secrets using valid vault expressions - i.e. expressions that identify the key that should be
+     * used to retrieve the secret from the vault.
+     * <p/>
+     * Some of the keys used in this test exist in the test vault while others, despite being valid expressions, don't identify
+     * any secret in the vault. For the former, the test compares the obtained secret against the expected secret (using both
+     * the buffer and array representations of the secret) and then checks if the secrets have been overridden/destroyed after
+     * the try-wih-resources block. For the latter, the tests checks if an empty {@link Optional} has been returned by the
+     * transcriber.
+     *
+     * @throws Exception if an error occurs while running the test.
+     */
+    @Test
+    public void testGetCharSecretUsingValidExpressions() throws Exception {
+        CharBuffer secretBuffer = null;
+        char[] secretArray = null;
+
+        // attempt to obtain a secret using a proper vault expressions. The key may or may not exist in the vault, so we
+        // check both cases using the returned optional and comparing against the expected secret.
+        for (String key : validExpressions.keySet()) {
+            String expectedSecret = validExpressions.get(key);
+            try (VaultCharSecret secret = transcriber.getCharSecret(key)) {
+                Optional<CharBuffer> optional = secret.get();
+                Optional<char[]> optionalArray = secret.getAsArray();
+                if (expectedSecret != null) {
+                    Assert.assertTrue(optional.isPresent());
+                    secretBuffer = optional.get();
+                    Assert.assertArrayEquals(expectedSecret.toCharArray(), secretBuffer.array());
+                    Assert.assertTrue(optionalArray.isPresent());
+                    secretArray = optionalArray.get();
+                    Assert.assertArrayEquals(expectedSecret.toCharArray(), secretArray);
+                } else {
+                    Assert.assertFalse(optional.isPresent());
+                    Assert.assertFalse(optionalArray.isPresent());
+                }
+            }
+            // after the try-with-resources block the secret should have been overridden.
+            if (expectedSecret != null) {
+                Assert.assertFalse(Arrays.equals(expectedSecret.toCharArray(), secretBuffer.array()));
+                Assert.assertFalse(Arrays.equals(expectedSecret.toCharArray(), secretArray));
+            }
+        }
+    }
+
+    /**
+     * Tests the retrieval of char secrets using invalid vault expressions - i.e. expressions that identify the key that should be
+     * used to retrieve the secret from the vault. When the values supplied to the transcriber are not valid vault expressions
+     * the value itself is assumed to be the secret and is enclosed in the secret class that is returned. Thus this test
+     * checks if the returned secret matches the specified values (using both the buffer and array representation of the value)
+     * and then checks if the secrets have been overridden/destroyed after the try-wih-resources block.
+     *
+     * @throws Exception if an error occurs while running the test.
+     */
+    @Test
+    public void testGetCharSecretUsingInvalidExpressions() throws Exception {
+        CharBuffer secretBuffer;
+        char[] secretArray;
+
+        // attempt to obtain a secret using invalid vault expressions - the value itself should be returned as a byte buffer.
+        for (String value : this.invalidExpressions) {
+            try (VaultCharSecret secret = transcriber.getCharSecret(value)) {
+                Optional<CharBuffer> optional = secret.get();
+                Optional<char[]> optionalArray = secret.getAsArray();
+                Assert.assertTrue(optional.isPresent());
+                secretBuffer = optional.get();
+                Assert.assertArrayEquals(value.toCharArray(), secretBuffer.array());
+                Assert.assertTrue(optionalArray.isPresent());
+                secretArray = optionalArray.get();
+                Assert.assertArrayEquals(value.toCharArray(), secretArray);
+            }
+            // after the try-with-resources block the secret should have been overridden.
+            if (!value.isEmpty()) {
+                Assert.assertFalse(Arrays.equals(value.toCharArray(), secretBuffer.array()));
+                Assert.assertFalse(Arrays.equals(value.toCharArray(), secretArray));
+            }
+        }
+    }
+
+    /**
+     * Tests that a null vault expression always returns an empty secret.
+     *
+     * @throws Exception if an error occurs while running the test.
+     */
+    @Test
+    public void testGetCharSecretUsingNullExpression() throws Exception {
+        // check that a null expression results in an empty optional instance.
+        try (VaultCharSecret secret = transcriber.getCharSecret(null)) {
+            Assert.assertFalse(secret.get().isPresent());
+            Assert.assertFalse(secret.getAsArray().isPresent());
+        }
+    }
+
+    /**
+     * Tests the retrieval of string secrets using valid vault expressions - i.e. expressions that identify the key that should be
+     * used to retrieve the secret from the vault.
+     * <p/>
+     * Some of the keys used in this test exist in the test vault while others, despite being valid expressions, don't identify
+     * any secret in the vault. For the former, the test compares the obtained secret against the expected secret. For the latter,
+     * the tests checks if an empty {@link Optional} has been returned by the transcriber. Because strings are immutable,
+     * this test doesn't verify if the secrets have been destroyed after the try-with-resources block.
+     *
+     * @throws Exception if an error occurs while running the test.
+     */
+    @Test
+    public void testGetStringSecretUsingValidExpressions() throws Exception {
+
+        // attempt to obtain a secret using a proper vault expressions. The key may or may not exist in the vault, so we
+        // check both cases using the returned optional and comparing against the expected secret.
+        for (String key : validExpressions.keySet()) {
+            String expectedSecret = validExpressions.get(key);
+            try (VaultStringSecret secret = transcriber.getStringSecret(key)) {
+                Optional<String> optional = secret.get();
+                if (expectedSecret != null) {
+                    Assert.assertTrue(optional.isPresent());
+                    String secretString = optional.get();
+                    Assert.assertEquals(expectedSecret, secretString);
+                } else {
+                    Assert.assertFalse(optional.isPresent());
+                }
+            }
+        }
+    }
+
+    /**
+     * Tests the retrieval of string secrets using invalid vault expressions - i.e. expressions that identify the key that should be
+     * used to retrieve the secret from the vault. When the values supplied to the transcriber are not valid vault expressions
+     * the value itself is assumed to be the secret and is enclosed in the secret class that is returned. Thus this test
+     * checks if the returned secret matches the specified values. Again, due to the fact that strings are immutable, this test
+     * doesn't verify if the secrets have been destroyed after the try-with-resources block.
+     *
+     * @throws Exception if an error occurs while running the test.
+     */
+    @Test
+    public void testGetStringSecretUsingInvalidExpressions() throws Exception {
+
+        // attempt to obtain a secret using invalid vault expressions - the value itself should be returned as a byte buffer.
+        for (String value : invalidExpressions) {
+            try (VaultStringSecret secret = transcriber.getStringSecret(value)) {
+                Optional<String> optional = secret.get();
+                Assert.assertTrue(optional.isPresent());
+                String secretString = optional.get();
+                Assert.assertEquals(value, secretString);
+            }
+        }
+    }
+
+    /**
+     * Tests that a null vault expression always returns an empty secret.
+     *
+     * @throws Exception if an error occurs while running the test.
+     */
+    @Test
+    public void testGetStringSecretUsingNullExpression() throws Exception {
+        // check that a null expression results in an empty optional instance.
+        try (VaultStringSecret secret = transcriber.getStringSecret(null)) {
+            Assert.assertFalse(secret.get().isPresent());
+        }
+    }
+
+    /**
+     * Tests that when no {@link VaultProvider} is supplied to the transcriber it uses a default implementation that
+     * always returns empty secrets.
+     *
+     * @throws Exception if an error occurs while running the test.
+     */
+    @Test
+    public void testTranscriberWithNullProvider() throws Exception {
+        VaultTranscriber transcriber = new DefaultVaultTranscriber(null);
+        // none of the valid expressions identify a key in the default vault as it always returns empty secrets.
+        for (String key : validExpressions.keySet()) {
+            try (VaultRawSecret secret = transcriber.getRawSecret(key)) {
+                Assert.assertFalse(secret.get().isPresent());
+                Assert.assertFalse(secret.getAsArray().isPresent());
+            }
+        }
+        // for invalid expressions, the transcriber doesn't rely on the provider so it should encode the value itself.
+        for (String value : invalidExpressions) {
+            try (VaultStringSecret secret = transcriber.getStringSecret(value)) {
+                Optional<String> optional = secret.get();
+                Assert.assertTrue(optional.isPresent());
+                String secretString = optional.get();
+                Assert.assertEquals(value, secretString);
+            }
+        }
+    }
+
+    class TestVaultProvider implements VaultProvider {
+
+        private Map<String, byte[]> secrets = new HashMap<>();
+
+        TestVaultProvider() {
+            secrets.put("vault_key_1", "secret1".getBytes());
+            secrets.put("vault_key_2", "secret2".getBytes());
+        }
+
+        @Override
+        public VaultRawSecret obtainSecret(String vaultSecretId) {
+            if (secrets.containsKey(vaultSecretId)) {
+                return DefaultVaultRawSecret.forBuffer(Optional.of(ByteBuffer.wrap(secrets.get(vaultSecretId))));
+            }
+            else {
+                return DefaultVaultRawSecret.forBuffer(Optional.empty());
+            }
+        }
+
+        @Override
+        public void close() {
+            // nothing to do
+        }
+    }
+}

testsuite/integration-arquillian/servers/auth-server/jboss/common/jboss-cli/keycloak-server-subsystem.cli

@@ -12,6 +12,10 @@ echo ** Adding login-protocol spi **
 /subsystem=keycloak-server/spi=login-protocol/:add
 /subsystem=keycloak-server/spi=login-protocol/provider=saml/:add(enabled=true,properties={knownProtocols => "[\"http=${auth.server.http.port}\",\"https=${auth.server.https.port}\"]"})
 
+echo ** Adding vault spi **
+/subsystem=keycloak-server/spi=vault/:add
+/subsystem=keycloak-server/spi=vault/provider=plaintext/:add(enabled=true,properties={dir => "${jboss.home.dir}/standalone/configuration/vault"})
+
 echo ** Adding theme modules **
 /subsystem=keycloak-server/theme=defaults/:write-attribute(name=modules,value=[org.keycloak.testsuite.integration-arquillian-testsuite-providers])
 

testsuite/integration-arquillian/servers/auth-server/jboss/common/vault/master_smtp__key

@@ -0,0 +1 @@
+secure_master_smtp_secret

testsuite/integration-arquillian/servers/auth-server/jboss/common/vault/test_smtp__key

@@ -0,0 +1 @@
+secure_test_smtp_secret

testsuite/integration-arquillian/servers/auth-server/jboss/pom.xml

@@ -206,6 +206,25 @@
                                     </resources>
                                 </configuration>
                             </execution>
+                            <execution>
+                                <id>copy-vault</id>
+                                <phase>process-resources</phase>
+                                <goals>
+                                    <goal>copy-resources</goal>
+                                </goals>
+                                <configuration>
+                                    <outputDirectory>${auth.server.home}/standalone/configuration/vault</outputDirectory>
+                                    <resources>
+                                        <resource>
+                                            <directory>${common.resources}/vault</directory>
+                                            <includes>
+                                                <include>master_smtp__key</include>
+                                                <include>test_smtp__key</include>
+                                            </includes>
+                                        </resource>
+                                    </resources>
+                                </configuration>
+                            </execution>
                         </executions>
                     </plugin>
                     <plugin>

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/vault/KeycloakVaultTest.java

@@ -0,0 +1,101 @@
+/*
+ * Copyright 2019 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.testsuite.vault;
+
+import java.lang.ref.WeakReference;
+import java.util.List;
+import java.util.Optional;
+
+import org.jboss.arquillian.container.test.api.Deployment;
+import org.jboss.shrinkwrap.api.spec.WebArchive;
+import org.junit.Assert;
+import org.junit.Test;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.representations.idm.RealmRepresentation;
+import org.keycloak.testsuite.AbstractKeycloakTest;
+import org.keycloak.testsuite.runonserver.RunOnServer;
+import org.keycloak.testsuite.runonserver.RunOnServerDeployment;
+import org.keycloak.testsuite.utils.io.IOUtil;
+import org.keycloak.vault.VaultStringSecret;
+import org.keycloak.vault.VaultTranscriber;
+
+/**
+ * Tests the usage of the {@link VaultTranscriber} on the server side. The tests attempt to obtain the transcriber from
+ * the session and then use it to obtain secrets from the configured provider.
+ *
+ * @author <a href="mailto:sguilhen@redhat.com">Stefan Guilhen</a>
+ */
+public class KeycloakVaultTest extends AbstractKeycloakTest {
+
+    @Deployment
+    public static WebArchive deploy() {
+        return RunOnServerDeployment.create();
+    }
+
+    @Override
+    public void addTestRealms(List<RealmRepresentation> testRealms) {
+        testRealms.add(IOUtil.loadRealm("/testrealm.json"));
+    }
+
+    @Test
+    public void testKeycloakVault() throws Exception {
+        // run the test in two different realms to test the provider's ability to retrieve secrets with the same key in different realms.
+        testingClient.server().run(new KeycloakVaultServerTest("${vault.smtp_key}", "secure_master_smtp_secret"));
+        testingClient.server("test").run(new KeycloakVaultServerTest("${vault.smtp_key}", "secure_test_smtp_secret"));
+    }
+
+    static class KeycloakVaultServerTest implements RunOnServer {
+
+        private String testKey;
+        private String expectedSecret;
+
+        public KeycloakVaultServerTest(final String key, final String expectedSecret) {
+            this.testKey = key;
+            this.expectedSecret = expectedSecret;
+        }
+
+        @Override
+        public void run(KeycloakSession session) {
+            VaultTranscriber transcriber = session.vault();
+            Assert.assertNotNull(transcriber);
+
+            // use the transcriber to obtain a secret from the vault.
+            try (VaultStringSecret secret = transcriber.getStringSecret(testKey)){
+                Optional<String> optional = secret.get();
+                Assert.assertTrue(optional.isPresent());
+                String secretString = optional.get();
+                Assert.assertEquals(expectedSecret, secretString);
+            }
+
+            // try obtaining a secret using a key that does not exist in the vault.
+            String invalidEntry = "${vault.invalid_entry}";
+            try (VaultStringSecret secret = transcriber.getStringSecret(invalidEntry)) {
+                Optional<String> optional = secret.get();
+                Assert.assertFalse(optional.isPresent());
+            }
+
+            // invoke the transcriber using a string that is not a vault expression.
+            try (VaultStringSecret secret = transcriber.getStringSecret("mysecret")) {
+                Optional<String> optional = secret.get();
+                Assert.assertTrue(optional.isPresent());
+                String secretString = optional.get();
+                Assert.assertEquals("mysecret", secretString);
+            }
+        }
+    }
+}

testsuite/integration-arquillian/tests/base/src/test/resources/META-INF/keycloak-server.json

@@ -184,5 +184,13 @@
             "sslCertChainPrefix": "x-ssl-client-cert-chain",
             "certificateChainLength": 1
         }
+    },
+
+    "vault": {
+        "provider": "${keycloak.vault.provider:plaintext}",
+        "plaintext": {
+            "dir": "src/test/resources/vault",
+            "disabled": false
+        }
     }
 }

testsuite/integration-arquillian/tests/base/src/test/resources/vault/master_smtp__key

@@ -0,0 +1 @@
+secure_master_smtp_secret

testsuite/integration-arquillian/tests/base/src/test/resources/vault/test_smtp__key

@@ -0,0 +1 @@
+secure_test_smtp_secret