diff --git a/server-spi-private/src/main/java/org/keycloak/vault/VaultProvider.java b/server-spi-private/src/main/java/org/keycloak/vault/VaultProvider.java index 259c84aead..91ad41a24d 100644 --- a/server-spi-private/src/main/java/org/keycloak/vault/VaultProvider.java +++ b/server-spi-private/src/main/java/org/keycloak/vault/VaultProvider.java @@ -28,7 +28,7 @@ public interface VaultProvider extends Provider { * Retrieves a secret from vault. The implementation should respect * at least the realm ID to separate the secrets within the vault. * If the secret is retrieved successfully, it is returned; - * otherwise this method results into an empty {@link VaultRawSecret#getRawSecret()}. + * otherwise this method results into an empty {@link VaultRawSecret#get()}. * * This method is intended to be used within a try-with-resources block so that * the secret is destroyed immediately after use. @@ -42,7 +42,7 @@ public interface VaultProvider extends Provider { * * @return Always a non-{@code null} value with the raw secret. * Within the returned value, the secret or {@code null} is stored in the - * {@link VaultRawSecret#getRawSecret()} return value if the secret was successfully + * {@link VaultRawSecret#get()} return value if the secret was successfully * resolved, or an empty {@link java.util.Optional} if the secret has not been found in the vault. */ VaultRawSecret obtainSecret(String vaultSecretId); diff --git a/server-spi/src/main/java/org/keycloak/models/KeycloakSession.java b/server-spi/src/main/java/org/keycloak/models/KeycloakSession.java index 19773dcaf9..3015c60d51 100755 --- a/server-spi/src/main/java/org/keycloak/models/KeycloakSession.java +++ b/server-spi/src/main/java/org/keycloak/models/KeycloakSession.java @@ -22,6 +22,7 @@ import org.keycloak.models.cache.UserCache; import org.keycloak.provider.Provider; import org.keycloak.sessions.AuthenticationSessionProvider; import org.keycloak.storage.federated.UserFederatedStorageProvider; +import org.keycloak.vault.VaultTranscriber; import java.util.Set; @@ -196,4 +197,8 @@ public interface KeycloakSession { */ TokenManager tokens(); + /** + * Vault transcriber + */ + VaultTranscriber vault(); } diff --git a/server-spi/src/main/java/org/keycloak/vault/VaultCharSecret.java b/server-spi/src/main/java/org/keycloak/vault/VaultCharSecret.java new file mode 100644 index 0000000000..e4fd2b3265 --- /dev/null +++ b/server-spi/src/main/java/org/keycloak/vault/VaultCharSecret.java @@ -0,0 +1,51 @@ +/* + * Copyright 2019 Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.keycloak.vault; + +import java.nio.CharBuffer; +import java.util.Optional; + +/** + * A {@link CharBuffer} based representation of the secret obtained from the vault that supports automated cleanup of memory. + * + * @author Stefan Guilhen + */ +public interface VaultCharSecret extends AutoCloseable { + + /** + * Returns the secret enclosed in a {@link CharBuffer}. + * + * @return If the secret was successfully resolved by vault, returns an {@link Optional} containing the value returned + * by the vault as a {@link CharBuffer} (a valid value can be {@code null}), or an empty {@link Optional} + */ + Optional get(); + + /** + * Returns the secret in its {@code char[]} form. + * + * @return If the secret was successfully resolved by vault, returns an {@link Optional} containing the value returned + * by the vault as a {@code char[]} (a valid value can be {@code null}), or an empty {@link Optional}. + */ + Optional getAsArray(); + + /** + * Destroys the secret in memory by e.g. overwriting it with random garbage. + */ + @Override + void close(); +} \ No newline at end of file diff --git a/server-spi-private/src/main/java/org/keycloak/vault/VaultRawSecret.java b/server-spi/src/main/java/org/keycloak/vault/VaultRawSecret.java similarity index 78% rename from server-spi-private/src/main/java/org/keycloak/vault/VaultRawSecret.java rename to server-spi/src/main/java/org/keycloak/vault/VaultRawSecret.java index e417c608db..7acc1159a2 100644 --- a/server-spi-private/src/main/java/org/keycloak/vault/VaultRawSecret.java +++ b/server-spi/src/main/java/org/keycloak/vault/VaultRawSecret.java @@ -32,7 +32,15 @@ public interface VaultRawSecret extends AutoCloseable { * an {@link Optional} containing the value returned by the vault * (a valid value can be {@code null}), or an empty {@link Optional} */ - Optional getRawSecret(); + Optional get(); + + /** + * Returns the raw secret bytes in {@code byte[]} form. + * @return If the secret was successfully resolved by vault, returns + * an {@link Optional} containing the value returned by the vault + * (a valid value can be {@code null}), or an empty {@link Optional} + */ + Optional getAsArray(); /** * Destroys the secret in memory by e.g. overwriting it with random garbage. diff --git a/server-spi/src/main/java/org/keycloak/vault/VaultStringSecret.java b/server-spi/src/main/java/org/keycloak/vault/VaultStringSecret.java new file mode 100644 index 0000000000..512d149450 --- /dev/null +++ b/server-spi/src/main/java/org/keycloak/vault/VaultStringSecret.java @@ -0,0 +1,44 @@ +/* + * Copyright 2019 Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.keycloak.vault; + +import java.util.Optional; + +/** + * A {@link String} based representation of the secret obtained from the vault that supports automated cleanup of memory. + * In this case, due to the immutable nature of strings, the cleanup should consist in releasing any references to the + * secret string so it can be disposed by the GC as soon as possible. + * + * @author Stefan Guilhen + */ +public interface VaultStringSecret extends AutoCloseable { + + /** + * Returns the secret represented as a {@link String}. + * @return If the secret was successfully resolved by vault, returns an {@link Optional} containing the value returned + * by the vault as a {@link String} (a valid value can be {@code null}), or an empty {@link Optional} + */ + Optional get(); + + /** + * Destroys the secret in memory by e.g. overwriting it with random garbage or release references in case of immutable + * secrets. + */ + @Override + void close(); +} diff --git a/server-spi/src/main/java/org/keycloak/vault/VaultTranscriber.java b/server-spi/src/main/java/org/keycloak/vault/VaultTranscriber.java new file mode 100644 index 0000000000..686938a8bf --- /dev/null +++ b/server-spi/src/main/java/org/keycloak/vault/VaultTranscriber.java @@ -0,0 +1,88 @@ +/* + * Copyright 2019 Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.keycloak.vault; + +import java.lang.ref.WeakReference; +import java.nio.CharBuffer; +import java.util.Optional; + +/** + * A facade to the configured vault provider that exposes utility methods for obtaining the vault secrets in different + * formats (such as {@link VaultRawSecret}, {@link VaultCharSecret} or {@link VaultStringSecret}). + * + * @see VaultRawSecret + * @see VaultCharSecret + * + * @author Stefan Guilhen + */ +public interface VaultTranscriber { + + /** + * Obtains the raw secret from the vault that matches the entry in the specified value string. The value must follow + * the format {@code ${vault.}} where {@code } identifies the entry in the vault. If the value doesn't follow + * the vault expression format, it is assumed to be the secret itself and is encoded into a {@link VaultRawSecret}. + *

+ * The returned {@link VaultRawSecret} extends {@link AutoCloseable} and it is strongly recommended that it is used in + * try-with-resources blocks to ensure the raw secret is overridden (destroyed) when the calling code is finished using + * it. + * + * @param value a {@link String} that might be a vault expression containing a vault entry key. + * @return a {@link VaultRawSecret} representing the secret that was read from the vault. If the specified value is not + * a vault expression then the returned secret is the value itself encoded as a {@link VaultRawSecret}. + */ + VaultRawSecret getRawSecret(final String value); + + /** + * Obtains the secret represented as a {@link VaultCharSecret} from the vault that matches the entry in the specified + * value string. The value must follow the format {@code ${vault.}} where {@code } identifies the entry in + * the vault. If the value doesn't follow the vault expression format, it is assumed to be the secret itself and is + * encoded into a {@link VaultCharSecret}. + *

+ * The returned {@link VaultCharSecret} extends {@link AutoCloseable} and it is strongly recommended that it is used in + * try-with-resources blocks to ensure the raw secret is overridden (destroyed) when the calling code is finished using + * it. + * + * @param value a {@link String} that might be a vault expression containing a vault entry key. + * @return a {@link VaultRawSecret} representing the secret that was read from the vault. If the specified value is not + * a vault expression then the returned secret is the value itself encoded as a {@link VaultRawSecret}. + */ + VaultCharSecret getCharSecret(final String value); + + /** + * Obtains the secret represented as a {@link String} from the vault that matches the entry in the specified value. + * The value must follow the format {@code ${vault.}} where {@code } identifies the entry in the vault. If + * the value doesn't follow the vault expression format, it is assumed to be the secret itself. + *

+ * Due to the immutable nature of strings and the way the JVM handles them internally, implementations that keep a reference + * to the secret string might consider doing so using a {@link WeakReference} that can be cleared in the {@link AutoCloseable#close()} + * method. Being immutable, such strings cannot be overridden (destroyed) by the implementation, but using a {@link WeakReference} + * guarantees that at least no hard references to the secret are held by the implementation class itself (which would + * prevent proper GC disposal of the secrets). + *

+ * WARNING: It is strongly recommended that callers of this method use the returned secret in try-with-resources + * blocks and they should strive not to keep hard references to the enclosed secret string for any longer than necessary + * so that the secret becomes available for GC as soon as possible. These measures help shorten the window of time when + * the secret strings are readable from memory. + * + * @param value a {@link String} that might be a vault expression containing a vault entry key. + * @return a {@link VaultStringSecret} representing the secret that was read from the vault. If the specified value is not + * a vault expression then the returned secret is the value itself encoded as a {@link VaultStringSecret}. + */ + VaultStringSecret getStringSecret(final String value); + +} \ No newline at end of file diff --git a/services/src/main/java/org/keycloak/services/DefaultKeycloakSession.java b/services/src/main/java/org/keycloak/services/DefaultKeycloakSession.java index f280ea73ac..5d74a07f2e 100644 --- a/services/src/main/java/org/keycloak/services/DefaultKeycloakSession.java +++ b/services/src/main/java/org/keycloak/services/DefaultKeycloakSession.java @@ -42,6 +42,9 @@ import org.keycloak.storage.ClientStorageManager; import org.keycloak.storage.UserStorageManager; import org.keycloak.storage.federated.UserFederatedStorageProvider; import org.keycloak.theme.DefaultThemeManager; +import org.keycloak.vault.DefaultVaultTranscriber; +import org.keycloak.vault.VaultProvider; +import org.keycloak.vault.VaultTranscriber; import java.util.HashMap; import java.util.HashSet; @@ -71,6 +74,7 @@ public class DefaultKeycloakSession implements KeycloakSession { private KeyManager keyManager; private ThemeManager themeManager; private TokenManager tokenManager; + private VaultTranscriber vaultTranscriber; public DefaultKeycloakSession(DefaultKeycloakSessionFactory factory) { this.factory = factory; @@ -302,6 +306,14 @@ public class DefaultKeycloakSession implements KeycloakSession { return tokenManager; } + @Override + public VaultTranscriber vault() { + if (this.vaultTranscriber == null) { + this.vaultTranscriber = new DefaultVaultTranscriber(this.getProvider(VaultProvider.class)); + } + return this.vaultTranscriber; + } + public void close() { for (Provider p : providers.values()) { try { diff --git a/services/src/main/java/org/keycloak/vault/DefaultVaultCharSecret.java b/services/src/main/java/org/keycloak/vault/DefaultVaultCharSecret.java new file mode 100644 index 0000000000..cc9e3363ac --- /dev/null +++ b/services/src/main/java/org/keycloak/vault/DefaultVaultCharSecret.java @@ -0,0 +1,95 @@ +/* + * Copyright 2019 Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.keycloak.vault; + +import java.nio.CharBuffer; +import java.util.Optional; +import java.util.concurrent.ThreadLocalRandom; + +/** + * Default {@link VaultCharSecret} implementation based on {@link CharBuffer}. + * + * @author Stefan Guilhen + */ +public class DefaultVaultCharSecret implements VaultCharSecret { + + private static final VaultCharSecret EMPTY_VAULT_SECRET = new VaultCharSecret() { + @Override + public Optional get() { + return Optional.empty(); + } + + @Override + public Optional getAsArray() { + return Optional.empty(); + } + + @Override + public void close() { + } + }; + + public static VaultCharSecret forBuffer(Optional buffer) { + if (buffer == null || ! buffer.isPresent()) { + return EMPTY_VAULT_SECRET; + } + return new DefaultVaultCharSecret(buffer.get()); + } + + private final CharBuffer buffer; + + private char[] secretArray; + + private DefaultVaultCharSecret(final CharBuffer buffer) { + this.buffer = buffer; + } + + @Override + public Optional get() { + return Optional.of(this.buffer); + } + + @Override + public Optional getAsArray() { + if (this.secretArray == null) { + // initialize internal array on demand. + if (this.buffer.hasArray()) { + this.secretArray = buffer.array(); + } else { + secretArray = new char[buffer.capacity()]; + buffer.get(secretArray); + } + } + return Optional.of(this.secretArray); + } + + @Override + public void close() { + if (this.buffer.hasArray()) { + char[] internalArray = this.buffer.array(); + for (int i = 0; i < internalArray.length; i++) { + internalArray[i] = (char) ThreadLocalRandom.current().nextInt(); + } + } else if (this.secretArray != null) { + for (int i = 0; i < this.secretArray.length; i++) { + this.secretArray[i] = (char) ThreadLocalRandom.current().nextInt(); + } + } + this.buffer.clear(); + } +} \ No newline at end of file diff --git a/services/src/main/java/org/keycloak/vault/DefaultVaultRawSecret.java b/services/src/main/java/org/keycloak/vault/DefaultVaultRawSecret.java index 7528b8c70b..0310ea84a2 100644 --- a/services/src/main/java/org/keycloak/vault/DefaultVaultRawSecret.java +++ b/services/src/main/java/org/keycloak/vault/DefaultVaultRawSecret.java @@ -28,7 +28,12 @@ public class DefaultVaultRawSecret implements VaultRawSecret { private static final VaultRawSecret EMPTY_VAULT_SECRET = new VaultRawSecret() { @Override - public Optional getRawSecret() { + public Optional get() { + return Optional.empty(); + } + + @Override + public Optional getAsArray() { return Optional.empty(); } @@ -39,6 +44,8 @@ public class DefaultVaultRawSecret implements VaultRawSecret { private final ByteBuffer rawSecret; + private byte[] secretArray; + public static VaultRawSecret forBuffer(Optional buffer) { if (buffer == null || ! buffer.isPresent()) { return EMPTY_VAULT_SECRET; @@ -51,14 +58,30 @@ public class DefaultVaultRawSecret implements VaultRawSecret { } @Override - public Optional getRawSecret() { + public Optional get() { return Optional.of(this.rawSecret); } + @Override + public Optional getAsArray() { + if (this.secretArray == null) { + // initialize internal array on demand. + if (this.rawSecret.hasArray()) { + this.secretArray = this.rawSecret.array(); + } else { + secretArray = new byte[this.rawSecret.capacity()]; + this.rawSecret.get(secretArray); + } + } + return Optional.of(this.secretArray); + } + @Override public void close() { if (rawSecret.hasArray()) { ThreadLocalRandom.current().nextBytes(rawSecret.array()); + } else if (this.secretArray != null) { + ThreadLocalRandom.current().nextBytes(this.secretArray); } rawSecret.clear(); } diff --git a/services/src/main/java/org/keycloak/vault/DefaultVaultStringSecret.java b/services/src/main/java/org/keycloak/vault/DefaultVaultStringSecret.java new file mode 100644 index 0000000000..851cb33623 --- /dev/null +++ b/services/src/main/java/org/keycloak/vault/DefaultVaultStringSecret.java @@ -0,0 +1,46 @@ +package org.keycloak.vault; + +import java.lang.ref.WeakReference; +import java.util.Optional; + +/** + * Default {@link VaultCharSecret} implementation based on {@link String}. + * + * @author Stefan Guilhen + */ +public class DefaultVaultStringSecret implements VaultStringSecret { + + private static final VaultStringSecret EMPTY_VAULT_SECRET = new VaultStringSecret() { + @Override + public Optional get() { + return Optional.empty(); + } + + @Override + public void close() { + } + }; + + public static VaultStringSecret forString(Optional secret) { + if (secret == null || ! secret.isPresent()) { + return EMPTY_VAULT_SECRET; + } + return new DefaultVaultStringSecret(secret.get()); + } + + private String secret; + + private DefaultVaultStringSecret(final String secret) { + this.secret = secret; + } + + @Override + public Optional get() { + return Optional.of(this.secret); + } + + @Override + public void close() { + this.secret = null; + } +} diff --git a/services/src/main/java/org/keycloak/vault/DefaultVaultTranscriber.java b/services/src/main/java/org/keycloak/vault/DefaultVaultTranscriber.java new file mode 100644 index 0000000000..3fa1c11c1e --- /dev/null +++ b/services/src/main/java/org/keycloak/vault/DefaultVaultTranscriber.java @@ -0,0 +1,116 @@ +/* + * Copyright 2019 Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.keycloak.vault; + +import java.lang.ref.WeakReference; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Default {@link VaultTranscriber} implementation that uses the configured {@link VaultProvider} to obtain raw secrets + * and convert them into other types. By default, the {@link VaultProvider} provides raw secrets through a {@link ByteBuffer}. + * This class offers methods to convert the raw secrets into other types (such as {@link VaultCharSecret} or {@link WeakReference}). + * + * @see VaultRawSecret + * @see VaultCharSecret + * + * @author Stefan Guilhen + */ +public class DefaultVaultTranscriber implements VaultTranscriber { + + private static final Pattern pattern = Pattern.compile("^\\$\\{vault\\.(.+?)}$"); + + private final VaultProvider provider; + + public DefaultVaultTranscriber(final VaultProvider provider) { + if (provider == null) { + this.provider = new VaultProvider() { + @Override + public VaultRawSecret obtainSecret(String vaultSecretId) { + return DefaultVaultRawSecret.forBuffer(null); + } + + @Override + public void close() { + } + }; + } else { + this.provider = provider; + } + } + + @Override + public VaultRawSecret getRawSecret(final String value) { + String entryId = this.getVaultEntryKey(value); + if (entryId != null) { + // we have a valid ${vault.} string, use the provider to retrieve the entry. + return this.provider.obtainSecret(entryId); + } else { + // not a vault expression - encode the value itself as a byte buffer. + ByteBuffer buffer = value != null ? ByteBuffer.wrap(value.getBytes(StandardCharsets.UTF_8)) : null; + return DefaultVaultRawSecret.forBuffer(Optional.ofNullable(buffer)); + } + } + + @Override + public VaultCharSecret getCharSecret(final String value) { + // obtain the raw secret and convert it into a char secret. + try (VaultRawSecret rawSecret = this.getRawSecret(value)) { + if (!rawSecret.get().isPresent()) { + return DefaultVaultCharSecret.forBuffer(Optional.empty()); + } + ByteBuffer rawSecretBuffer = rawSecret.get().get(); + CharBuffer charSecretBuffer = StandardCharsets.UTF_8.decode(rawSecretBuffer); + return DefaultVaultCharSecret.forBuffer(Optional.of(charSecretBuffer)); + } + } + + @Override + public VaultStringSecret getStringSecret(final String value) { + // obtain the raw secret and convert it into a string string. + try (VaultRawSecret rawSecret = this.getRawSecret(value)) { + if (!rawSecret.get().isPresent()) { + return DefaultVaultStringSecret.forString(Optional.empty()); + } + ByteBuffer rawSecretBuffer = rawSecret.get().get(); + return DefaultVaultStringSecret.forString(Optional.of(StandardCharsets.UTF_8.decode(rawSecretBuffer).toString())); + } + } + + /** + * Obtains the vault entry key from the specified value if the value is a valid {@code ${vault.}} expression. + * For example, calling this method with the {@code ${vault.smtp_secret}} argument results in the string {@code smtp_secret} + * being returned. + * + * @param value a {@code String} that might contain a vault entry key. + * @return the extracted entry key if the value follows the {@code ${vault.}} format; null otherwise. + */ + private String getVaultEntryKey(final String value) { + if (value != null) { + Matcher matcher = pattern.matcher(value); + if (matcher.matches()) { + return matcher.group(1); + } + } + return null; + } +} diff --git a/services/src/test/java/org/keycloak/vault/PlainTextVaultProviderTest.java b/services/src/test/java/org/keycloak/vault/PlainTextVaultProviderTest.java index d514108b82..ac05657937 100644 --- a/services/src/test/java/org/keycloak/vault/PlainTextVaultProviderTest.java +++ b/services/src/test/java/org/keycloak/vault/PlainTextVaultProviderTest.java @@ -29,7 +29,7 @@ public class PlainTextVaultProviderTest { //then assertNotNull(secret1); - assertNotNull(secret1.getRawSecret().get()); + assertNotNull(secret1.get().get()); assertThat(secret1, secretContains("secret1")); } @@ -43,7 +43,7 @@ public class PlainTextVaultProviderTest { //then assertNotNull(secret1); - assertNotNull(secret1.getRawSecret().get()); + assertNotNull(secret1.get().get()); assertThat(secret1, secretContains("underscore_secret1")); } @@ -57,7 +57,7 @@ public class PlainTextVaultProviderTest { //then assertNotNull(secret); - assertFalse(secret.getRawSecret().isPresent()); + assertFalse(secret.get().isPresent()); } @Test @@ -70,7 +70,7 @@ public class PlainTextVaultProviderTest { //then assertNotNull(secret); - assertFalse(secret.getRawSecret().isPresent()); + assertFalse(secret.get().isPresent()); } @Test @@ -93,12 +93,12 @@ public class PlainTextVaultProviderTest { Files.write(temporarySecretFile, "secret1".getBytes()); try (VaultRawSecret secret1 = provider.obtainSecret(secretName)) { - secret1AsString = StandardCharsets.UTF_8.decode(secret1.getRawSecret().get()).toString(); + secret1AsString = StandardCharsets.UTF_8.decode(secret1.get().get()).toString(); } Files.write(temporarySecretFile, "secret2".getBytes()); try (VaultRawSecret secret2 = provider.obtainSecret(secretName)) { - secret2AsString = StandardCharsets.UTF_8.decode(secret2.getRawSecret().get()).toString(); + secret2AsString = StandardCharsets.UTF_8.decode(secret2.get().get()).toString(); } //then diff --git a/services/src/test/java/org/keycloak/vault/SecretContains.java b/services/src/test/java/org/keycloak/vault/SecretContains.java index e3718388c4..49919bd236 100644 --- a/services/src/test/java/org/keycloak/vault/SecretContains.java +++ b/services/src/test/java/org/keycloak/vault/SecretContains.java @@ -19,7 +19,7 @@ public class SecretContains extends TypeSafeMatcher { @Override protected boolean matchesSafely(VaultRawSecret secret) { - String convertedSecret = StandardCharsets.UTF_8.decode(secret.getRawSecret().get()).toString(); + String convertedSecret = StandardCharsets.UTF_8.decode(secret.get().get()).toString(); return thisVaultAsString.equals(convertedSecret); } diff --git a/services/src/test/java/org/keycloak/vault/VaultTranscriberTest.java b/services/src/test/java/org/keycloak/vault/VaultTranscriberTest.java new file mode 100644 index 0000000000..14b2812771 --- /dev/null +++ b/services/src/test/java/org/keycloak/vault/VaultTranscriberTest.java @@ -0,0 +1,361 @@ +/* + * Copyright 2019 Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.keycloak.vault; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Tests for the {@link DefaultVaultTranscriber} implementation. + * + * @author Stefan Guilhen + */ +public class VaultTranscriberTest { + + private final VaultTranscriber transcriber = new DefaultVaultTranscriber(new TestVaultProvider()); + + private static Map validExpressions; + + private static String[] invalidExpressions; + + @BeforeClass + public static void init() { + validExpressions = new HashMap<>(); + // expressions with keys that exist in the vault. + validExpressions.put("${vault.vault_key_1}", "secret1"); + validExpressions.put("${vault.vault_key_2}", "secret2"); + // expressions with keys that don't exist in the vault. + validExpressions.put("${vault.invalid_key}", null); + validExpressions.put("${vault.${.id-!@#$%^&*_()}}", null); + // invalid expressions. + invalidExpressions = new String[]{"${vault.}","$vault.id}", "{vault.id}", "${vault.id", "${vaultid}", ""}; + + } + + /** + * Tests the retrieval of raw secrets using valid vault expressions - i.e. expressions that identify the key that should be + * used to retrieve the secret from the vault. + *

+ * Some of the keys used in this test exist in the test vault while others, despite being valid expressions, don't identify + * any secret in the vault. For the former, the test compares the obtained secret against the expected secret (using both + * the buffer and array representations of the secret) and then checks if the secrets have been overridden/destroyed after + * the try-wih-resources block. For the latter, the tests checks if an empty {@link Optional} has been returned by the + * transcriber. + * + * @throws Exception if an error occurs while running the test. + */ + @Test + public void testGetRawSecretUsingValidExpressions() throws Exception { + ByteBuffer secretBuffer = null; + byte[] secretArray = null; + + // attempt to obtain a secret using a proper vault expressions. The key may or may not exist in the vault, so we + // check both cases using the returned optional and comparing against the expected secret. + for (String key : validExpressions.keySet()) { + String expectedSecret = validExpressions.get(key); + try (VaultRawSecret secret = transcriber.getRawSecret(key)) { + Optional optional = secret.get(); + Optional optionalArray = secret.getAsArray(); + if (expectedSecret != null) { + Assert.assertTrue(optional.isPresent()); + secretBuffer = optional.get(); + Assert.assertArrayEquals(expectedSecret.getBytes(StandardCharsets.UTF_8), secretBuffer.array()); + Assert.assertTrue(optionalArray.isPresent()); + secretArray = optionalArray.get(); + Assert.assertArrayEquals(expectedSecret.getBytes(StandardCharsets.UTF_8), secretArray); + } else { + Assert.assertFalse(optional.isPresent()); + Assert.assertFalse(optionalArray.isPresent()); + } + } + // after the try-with-resources block the secret should have been overridden. + if (expectedSecret != null) { + Assert.assertFalse(Arrays.equals(expectedSecret.getBytes(StandardCharsets.UTF_8), secretBuffer.array())); + Assert.assertFalse(Arrays.equals(expectedSecret.getBytes(StandardCharsets.UTF_8), secretArray)); + } + } + } + + /** + * Tests the retrieval of raw secrets using invalid vault expressions - i.e. expressions that identify the key that should be + * used to retrieve the secret from the vault. When the values supplied to the transcriber are not valid vault expressions + * the value itself is assumed to be the secret and is enclosed in the secret class that is returned. Thus this test + * checks if the returned secret matches the specified values (using both the buffer and array representation of the value) + * and then checks if the secrets have been overridden/destroyed after the try-wih-resources block. + * + * @throws Exception if an error occurs while running the test. + */ + @Test + public void testGetRawSecretUsingInvalidExpressions() throws Exception { + ByteBuffer secretBuffer; + byte[] secretArray; + + // attempt to obtain a secret using invalid vault expressions - the value itself should be returned as a byte buffer. + for (String value : this.invalidExpressions) { + try (VaultRawSecret secret = transcriber.getRawSecret(value)) { + Optional optional = secret.get(); + Optional optionalArray = secret.getAsArray(); + Assert.assertTrue(optional.isPresent()); + secretBuffer = optional.get(); + Assert.assertArrayEquals(value.getBytes(StandardCharsets.UTF_8), secretBuffer.array()); + Assert.assertTrue(optionalArray.isPresent()); + secretArray = optionalArray.get(); + Assert.assertArrayEquals(value.getBytes(StandardCharsets.UTF_8), secretArray); + } + // after the try-with-resources block the secret should have been overridden. + if (!value.isEmpty()) { + Assert.assertFalse(Arrays.equals(value.getBytes(StandardCharsets.UTF_8), secretBuffer.array())); + Assert.assertFalse(Arrays.equals(value.getBytes(StandardCharsets.UTF_8), secretArray)); + } + } + } + + /** + * Tests that a null vault expression always returns an empty secret. + * + * @throws Exception if an error occurs while running the test. + */ + @Test + public void testGetRawSecretUsingNullExpression() throws Exception { + // check that a null expression results in an empty optional instance. + try (VaultRawSecret secret = transcriber.getRawSecret(null)) { + Assert.assertFalse(secret.get().isPresent()); + Assert.assertFalse(secret.getAsArray().isPresent()); + } + } + + /** + * Tests the retrieval of char secrets using valid vault expressions - i.e. expressions that identify the key that should be + * used to retrieve the secret from the vault. + *

+ * Some of the keys used in this test exist in the test vault while others, despite being valid expressions, don't identify + * any secret in the vault. For the former, the test compares the obtained secret against the expected secret (using both + * the buffer and array representations of the secret) and then checks if the secrets have been overridden/destroyed after + * the try-wih-resources block. For the latter, the tests checks if an empty {@link Optional} has been returned by the + * transcriber. + * + * @throws Exception if an error occurs while running the test. + */ + @Test + public void testGetCharSecretUsingValidExpressions() throws Exception { + CharBuffer secretBuffer = null; + char[] secretArray = null; + + // attempt to obtain a secret using a proper vault expressions. The key may or may not exist in the vault, so we + // check both cases using the returned optional and comparing against the expected secret. + for (String key : validExpressions.keySet()) { + String expectedSecret = validExpressions.get(key); + try (VaultCharSecret secret = transcriber.getCharSecret(key)) { + Optional optional = secret.get(); + Optional optionalArray = secret.getAsArray(); + if (expectedSecret != null) { + Assert.assertTrue(optional.isPresent()); + secretBuffer = optional.get(); + Assert.assertArrayEquals(expectedSecret.toCharArray(), secretBuffer.array()); + Assert.assertTrue(optionalArray.isPresent()); + secretArray = optionalArray.get(); + Assert.assertArrayEquals(expectedSecret.toCharArray(), secretArray); + } else { + Assert.assertFalse(optional.isPresent()); + Assert.assertFalse(optionalArray.isPresent()); + } + } + // after the try-with-resources block the secret should have been overridden. + if (expectedSecret != null) { + Assert.assertFalse(Arrays.equals(expectedSecret.toCharArray(), secretBuffer.array())); + Assert.assertFalse(Arrays.equals(expectedSecret.toCharArray(), secretArray)); + } + } + } + + /** + * Tests the retrieval of char secrets using invalid vault expressions - i.e. expressions that identify the key that should be + * used to retrieve the secret from the vault. When the values supplied to the transcriber are not valid vault expressions + * the value itself is assumed to be the secret and is enclosed in the secret class that is returned. Thus this test + * checks if the returned secret matches the specified values (using both the buffer and array representation of the value) + * and then checks if the secrets have been overridden/destroyed after the try-wih-resources block. + * + * @throws Exception if an error occurs while running the test. + */ + @Test + public void testGetCharSecretUsingInvalidExpressions() throws Exception { + CharBuffer secretBuffer; + char[] secretArray; + + // attempt to obtain a secret using invalid vault expressions - the value itself should be returned as a byte buffer. + for (String value : this.invalidExpressions) { + try (VaultCharSecret secret = transcriber.getCharSecret(value)) { + Optional optional = secret.get(); + Optional optionalArray = secret.getAsArray(); + Assert.assertTrue(optional.isPresent()); + secretBuffer = optional.get(); + Assert.assertArrayEquals(value.toCharArray(), secretBuffer.array()); + Assert.assertTrue(optionalArray.isPresent()); + secretArray = optionalArray.get(); + Assert.assertArrayEquals(value.toCharArray(), secretArray); + } + // after the try-with-resources block the secret should have been overridden. + if (!value.isEmpty()) { + Assert.assertFalse(Arrays.equals(value.toCharArray(), secretBuffer.array())); + Assert.assertFalse(Arrays.equals(value.toCharArray(), secretArray)); + } + } + } + + /** + * Tests that a null vault expression always returns an empty secret. + * + * @throws Exception if an error occurs while running the test. + */ + @Test + public void testGetCharSecretUsingNullExpression() throws Exception { + // check that a null expression results in an empty optional instance. + try (VaultCharSecret secret = transcriber.getCharSecret(null)) { + Assert.assertFalse(secret.get().isPresent()); + Assert.assertFalse(secret.getAsArray().isPresent()); + } + } + + /** + * Tests the retrieval of string secrets using valid vault expressions - i.e. expressions that identify the key that should be + * used to retrieve the secret from the vault. + *

+ * Some of the keys used in this test exist in the test vault while others, despite being valid expressions, don't identify + * any secret in the vault. For the former, the test compares the obtained secret against the expected secret. For the latter, + * the tests checks if an empty {@link Optional} has been returned by the transcriber. Because strings are immutable, + * this test doesn't verify if the secrets have been destroyed after the try-with-resources block. + * + * @throws Exception if an error occurs while running the test. + */ + @Test + public void testGetStringSecretUsingValidExpressions() throws Exception { + + // attempt to obtain a secret using a proper vault expressions. The key may or may not exist in the vault, so we + // check both cases using the returned optional and comparing against the expected secret. + for (String key : validExpressions.keySet()) { + String expectedSecret = validExpressions.get(key); + try (VaultStringSecret secret = transcriber.getStringSecret(key)) { + Optional optional = secret.get(); + if (expectedSecret != null) { + Assert.assertTrue(optional.isPresent()); + String secretString = optional.get(); + Assert.assertEquals(expectedSecret, secretString); + } else { + Assert.assertFalse(optional.isPresent()); + } + } + } + } + + /** + * Tests the retrieval of string secrets using invalid vault expressions - i.e. expressions that identify the key that should be + * used to retrieve the secret from the vault. When the values supplied to the transcriber are not valid vault expressions + * the value itself is assumed to be the secret and is enclosed in the secret class that is returned. Thus this test + * checks if the returned secret matches the specified values. Again, due to the fact that strings are immutable, this test + * doesn't verify if the secrets have been destroyed after the try-with-resources block. + * + * @throws Exception if an error occurs while running the test. + */ + @Test + public void testGetStringSecretUsingInvalidExpressions() throws Exception { + + // attempt to obtain a secret using invalid vault expressions - the value itself should be returned as a byte buffer. + for (String value : invalidExpressions) { + try (VaultStringSecret secret = transcriber.getStringSecret(value)) { + Optional optional = secret.get(); + Assert.assertTrue(optional.isPresent()); + String secretString = optional.get(); + Assert.assertEquals(value, secretString); + } + } + } + + /** + * Tests that a null vault expression always returns an empty secret. + * + * @throws Exception if an error occurs while running the test. + */ + @Test + public void testGetStringSecretUsingNullExpression() throws Exception { + // check that a null expression results in an empty optional instance. + try (VaultStringSecret secret = transcriber.getStringSecret(null)) { + Assert.assertFalse(secret.get().isPresent()); + } + } + + /** + * Tests that when no {@link VaultProvider} is supplied to the transcriber it uses a default implementation that + * always returns empty secrets. + * + * @throws Exception if an error occurs while running the test. + */ + @Test + public void testTranscriberWithNullProvider() throws Exception { + VaultTranscriber transcriber = new DefaultVaultTranscriber(null); + // none of the valid expressions identify a key in the default vault as it always returns empty secrets. + for (String key : validExpressions.keySet()) { + try (VaultRawSecret secret = transcriber.getRawSecret(key)) { + Assert.assertFalse(secret.get().isPresent()); + Assert.assertFalse(secret.getAsArray().isPresent()); + } + } + // for invalid expressions, the transcriber doesn't rely on the provider so it should encode the value itself. + for (String value : invalidExpressions) { + try (VaultStringSecret secret = transcriber.getStringSecret(value)) { + Optional optional = secret.get(); + Assert.assertTrue(optional.isPresent()); + String secretString = optional.get(); + Assert.assertEquals(value, secretString); + } + } + } + + class TestVaultProvider implements VaultProvider { + + private Map secrets = new HashMap<>(); + + TestVaultProvider() { + secrets.put("vault_key_1", "secret1".getBytes()); + secrets.put("vault_key_2", "secret2".getBytes()); + } + + @Override + public VaultRawSecret obtainSecret(String vaultSecretId) { + if (secrets.containsKey(vaultSecretId)) { + return DefaultVaultRawSecret.forBuffer(Optional.of(ByteBuffer.wrap(secrets.get(vaultSecretId)))); + } + else { + return DefaultVaultRawSecret.forBuffer(Optional.empty()); + } + } + + @Override + public void close() { + // nothing to do + } + } +} \ No newline at end of file diff --git a/testsuite/integration-arquillian/servers/auth-server/jboss/common/jboss-cli/keycloak-server-subsystem.cli b/testsuite/integration-arquillian/servers/auth-server/jboss/common/jboss-cli/keycloak-server-subsystem.cli index e89d6e70c3..42bde9567d 100644 --- a/testsuite/integration-arquillian/servers/auth-server/jboss/common/jboss-cli/keycloak-server-subsystem.cli +++ b/testsuite/integration-arquillian/servers/auth-server/jboss/common/jboss-cli/keycloak-server-subsystem.cli @@ -12,6 +12,10 @@ echo ** Adding login-protocol spi ** /subsystem=keycloak-server/spi=login-protocol/:add /subsystem=keycloak-server/spi=login-protocol/provider=saml/:add(enabled=true,properties={knownProtocols => "[\"http=${auth.server.http.port}\",\"https=${auth.server.https.port}\"]"}) +echo ** Adding vault spi ** +/subsystem=keycloak-server/spi=vault/:add +/subsystem=keycloak-server/spi=vault/provider=plaintext/:add(enabled=true,properties={dir => "${jboss.home.dir}/standalone/configuration/vault"}) + echo ** Adding theme modules ** /subsystem=keycloak-server/theme=defaults/:write-attribute(name=modules,value=[org.keycloak.testsuite.integration-arquillian-testsuite-providers]) diff --git a/testsuite/integration-arquillian/servers/auth-server/jboss/common/vault/master_smtp__key b/testsuite/integration-arquillian/servers/auth-server/jboss/common/vault/master_smtp__key new file mode 100644 index 0000000000..e832461ce2 --- /dev/null +++ b/testsuite/integration-arquillian/servers/auth-server/jboss/common/vault/master_smtp__key @@ -0,0 +1 @@ +secure_master_smtp_secret \ No newline at end of file diff --git a/testsuite/integration-arquillian/servers/auth-server/jboss/common/vault/test_smtp__key b/testsuite/integration-arquillian/servers/auth-server/jboss/common/vault/test_smtp__key new file mode 100644 index 0000000000..3a4fa26153 --- /dev/null +++ b/testsuite/integration-arquillian/servers/auth-server/jboss/common/vault/test_smtp__key @@ -0,0 +1 @@ +secure_test_smtp_secret \ No newline at end of file diff --git a/testsuite/integration-arquillian/servers/auth-server/jboss/pom.xml b/testsuite/integration-arquillian/servers/auth-server/jboss/pom.xml index 9b277c273e..1a27583bff 100644 --- a/testsuite/integration-arquillian/servers/auth-server/jboss/pom.xml +++ b/testsuite/integration-arquillian/servers/auth-server/jboss/pom.xml @@ -206,6 +206,25 @@ + + copy-vault + process-resources + + copy-resources + + + ${auth.server.home}/standalone/configuration/vault + + + ${common.resources}/vault + + master_smtp__key + test_smtp__key + + + + + diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/vault/KeycloakVaultTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/vault/KeycloakVaultTest.java new file mode 100644 index 0000000000..e183b85255 --- /dev/null +++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/vault/KeycloakVaultTest.java @@ -0,0 +1,101 @@ +/* + * Copyright 2019 Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.keycloak.testsuite.vault; + +import java.lang.ref.WeakReference; +import java.util.List; +import java.util.Optional; + +import org.jboss.arquillian.container.test.api.Deployment; +import org.jboss.shrinkwrap.api.spec.WebArchive; +import org.junit.Assert; +import org.junit.Test; +import org.keycloak.models.KeycloakSession; +import org.keycloak.representations.idm.RealmRepresentation; +import org.keycloak.testsuite.AbstractKeycloakTest; +import org.keycloak.testsuite.runonserver.RunOnServer; +import org.keycloak.testsuite.runonserver.RunOnServerDeployment; +import org.keycloak.testsuite.utils.io.IOUtil; +import org.keycloak.vault.VaultStringSecret; +import org.keycloak.vault.VaultTranscriber; + +/** + * Tests the usage of the {@link VaultTranscriber} on the server side. The tests attempt to obtain the transcriber from + * the session and then use it to obtain secrets from the configured provider. + * + * @author Stefan Guilhen + */ +public class KeycloakVaultTest extends AbstractKeycloakTest { + + @Deployment + public static WebArchive deploy() { + return RunOnServerDeployment.create(); + } + + @Override + public void addTestRealms(List testRealms) { + testRealms.add(IOUtil.loadRealm("/testrealm.json")); + } + + @Test + public void testKeycloakVault() throws Exception { + // run the test in two different realms to test the provider's ability to retrieve secrets with the same key in different realms. + testingClient.server().run(new KeycloakVaultServerTest("${vault.smtp_key}", "secure_master_smtp_secret")); + testingClient.server("test").run(new KeycloakVaultServerTest("${vault.smtp_key}", "secure_test_smtp_secret")); + } + + static class KeycloakVaultServerTest implements RunOnServer { + + private String testKey; + private String expectedSecret; + + public KeycloakVaultServerTest(final String key, final String expectedSecret) { + this.testKey = key; + this.expectedSecret = expectedSecret; + } + + @Override + public void run(KeycloakSession session) { + VaultTranscriber transcriber = session.vault(); + Assert.assertNotNull(transcriber); + + // use the transcriber to obtain a secret from the vault. + try (VaultStringSecret secret = transcriber.getStringSecret(testKey)){ + Optional optional = secret.get(); + Assert.assertTrue(optional.isPresent()); + String secretString = optional.get(); + Assert.assertEquals(expectedSecret, secretString); + } + + // try obtaining a secret using a key that does not exist in the vault. + String invalidEntry = "${vault.invalid_entry}"; + try (VaultStringSecret secret = transcriber.getStringSecret(invalidEntry)) { + Optional optional = secret.get(); + Assert.assertFalse(optional.isPresent()); + } + + // invoke the transcriber using a string that is not a vault expression. + try (VaultStringSecret secret = transcriber.getStringSecret("mysecret")) { + Optional optional = secret.get(); + Assert.assertTrue(optional.isPresent()); + String secretString = optional.get(); + Assert.assertEquals("mysecret", secretString); + } + } + } +} \ No newline at end of file diff --git a/testsuite/integration-arquillian/tests/base/src/test/resources/META-INF/keycloak-server.json b/testsuite/integration-arquillian/tests/base/src/test/resources/META-INF/keycloak-server.json index 2a635d9305..02876fb8b5 100755 --- a/testsuite/integration-arquillian/tests/base/src/test/resources/META-INF/keycloak-server.json +++ b/testsuite/integration-arquillian/tests/base/src/test/resources/META-INF/keycloak-server.json @@ -184,5 +184,13 @@ "sslCertChainPrefix": "x-ssl-client-cert-chain", "certificateChainLength": 1 } + }, + + "vault": { + "provider": "${keycloak.vault.provider:plaintext}", + "plaintext": { + "dir": "src/test/resources/vault", + "disabled": false + } } } diff --git a/testsuite/integration-arquillian/tests/base/src/test/resources/vault/master_smtp__key b/testsuite/integration-arquillian/tests/base/src/test/resources/vault/master_smtp__key new file mode 100644 index 0000000000..e832461ce2 --- /dev/null +++ b/testsuite/integration-arquillian/tests/base/src/test/resources/vault/master_smtp__key @@ -0,0 +1 @@ +secure_master_smtp_secret \ No newline at end of file diff --git a/testsuite/integration-arquillian/tests/base/src/test/resources/vault/test_smtp__key b/testsuite/integration-arquillian/tests/base/src/test/resources/vault/test_smtp__key new file mode 100644 index 0000000000..3a4fa26153 --- /dev/null +++ b/testsuite/integration-arquillian/tests/base/src/test/resources/vault/test_smtp__key @@ -0,0 +1 @@ +secure_test_smtp_secret \ No newline at end of file