Support for KcReg and KcAdm CLI to use BCFIPS instead of BC on FIPS platforms

Closes #14968

.github/workflows/ci.yml

@@ -340,7 +340,7 @@ jobs:
           PARAMS["bcfips-nonapproved-pkcs12"]="-Pauth-server-quarkus,auth-server-fips140-2"
           # Tests in the package "forms" and some keystore related tests
           TESTGROUP["group1"]="-Dtest=org.keycloak.testsuite.forms.**,ClientAuthSignedJWTTest,CredentialsTest,JavaKeystoreKeyProviderTest,ServerInfoTest,UserFederationLdapConnectionTest,LDAPUserLoginTest"
-          TESTGROUP["group2"]="-Dtest=org.keycloak.testsuite.x509.**,MutualTLSClientTest,FAPI1Test,FAPICIBATest" # Tests for X.509 authentication with users and clients
+          TESTGROUP["group2"]="-Dtest=org.keycloak.testsuite.x509.**,MutualTLSClientTest,FAPI1Test,FAPICIBATest,KcRegTest,KcRegCreateTest,KcAdmTest,KcAdmCreateTest"   # Tests for X.509 authentication with users and clients and CLI tests
           
           ./mvnw clean install -nsu -B ${PARAMS["${{ matrix.server }}"]} ${TESTGROUP["${{ matrix.tests }}"]} -f testsuite/integration-arquillian/tests/base/pom.xml | misc/log/trimmer.sh
           

crypto/fips1402/src/test/java/org/keycloak/crypto/fips/test/FIPS1402SslTest.java

@@ -129,6 +129,7 @@ public class FIPS1402SslTest {
         List<String> supportedProtocols = Arrays.asList(context.getDefaultSSLParameters().getProtocols());
         List<String> supportedCiphers = Arrays.asList(engine.getSupportedCipherSuites());
 
+        logger.infof("SSLContext provider: %s, SSLContext class: %s", context.getProvider().getName(), context.getClass().getName());
         logger.infof("Enabled ciphersuites: %s", enabledCipherSuites.size());
         logger.infof("Supported protocols: %s", supportedProtocols);
         logger.infof("Supported ciphers size: %d", supportedCiphers.size());

docs/fips.md

@@ -20,9 +20,12 @@ running the unit tests below):
 ```
 cd $KEYCLOAK_HOME/bin
 export MAVEN_REPO_HOME=$HOME/.m2/repository
-cp $MAVEN_REPO_HOME/org/bouncycastle/bc-fips/1.0.2.3/bc-fips-1.0.2.3.jar ../providers/
+export BCFIPS_VERSION=1.0.2.3
-cp $MAVEN_REPO_HOME/org/bouncycastle/bctls-fips/1.0.12.2/bctls-fips-1.0.12.2.jar ../providers/
+export BCTLSFIPS_VERSION=1.0.12.2
-cp $MAVEN_REPO_HOME/org/bouncycastle/bcpkix-fips/1.0.5/bcpkix-fips-1.0.5.jar ../providers/
+export BCPKIXFIPS_VERSION=1.0.5
+cp $MAVEN_REPO_HOME/org/bouncycastle/bc-fips/$BCFIPS_VERSION/bc-fips-$BCFIPS_VERSION.jar ../providers/
+cp $MAVEN_REPO_HOME/org/bouncycastle/bctls-fips/$BCTLSFIPS_VERSION/bctls-fips-$BCTLSFIPS_VERSION.jar ../providers/
+cp $MAVEN_REPO_HOME/org/bouncycastle/bcpkix-fips/$BCPKIXFIPS_VERSION/bcpkix-fips-$BCPKIXFIPS_VERSION.jar ../providers/
 ```
 
 2) Now create either pkcs12 or bcfks keystore. The pkcs12 works just in BCFIPS non-approved mode.
@@ -114,6 +117,17 @@ Note that in approved mode, there are few limitations at the moment like for exa
 - Keystore/truststore must be of type bcfks due the both of `jks` and `pkcs12` don't work
 - Some warnings in the server.log at startup
 
+Run the CLI on the FIPS host
+----------------------------
+In case you want to run Client Registration CLI (`kcreg.sh/bat` script) or Admin CLI (`kcadm.sh/bat` script), it is needed
+that CLI will also use the BouncyCastle FIPS dependencies instead of plain BouncyCastle dependencies. To achieve this, you may copy the
+jars to the CLI library folder and that is enough. CLI tool will automatically use BCFIPS dependencies instead of plain BC when
+it detects that corresponding BCFIPS jars are present (see above for the versions used):
+```
+cp $MAVEN_REPO_HOME/org/bouncycastle/bc-fips/$BCFIPS_VERSION/bc-fips-$BCFIPS_VERSION.jar ../bin/client/lib/
+cp $MAVEN_REPO_HOME/org/bouncycastle/bctls-fips/$BCTLSFIPS_VERSION/bctls-fips-$BCTLSFIPS_VERSION.jar ../bin/client/lib/
+```
+
 Run the unit tests in the FIPS environment
 ------------------------------------------
 This instruction is about running automated tests on the FIPS enabled RHEL 8.6 system with the FIPS enabled OpenJDK 11.

integration/client-cli/admin-cli/pom.xml

@@ -38,10 +38,6 @@
             <groupId>org.keycloak</groupId>
             <artifactId>keycloak-core</artifactId>
         </dependency>
-        <dependency>
-            <groupId>org.keycloak</groupId>
-            <artifactId>${keycloak.crypto.artifactId}</artifactId>
-        </dependency>
         <dependency>
             <groupId>org.apache.httpcomponents</groupId>
             <artifactId>httpclient</artifactId>

integration/client-cli/admin-cli/src/main/bin/kcadm.bat

@@ -5,4 +5,4 @@ if "%OS%" == "Windows_NT" (
 ) else (
   set DIRNAME=.\
 )
-java %KC_OPTS% -cp "%DIRNAME%\client\keycloak-admin-cli-${project.version}.jar" org.keycloak.client.admin.cli.KcAdmMain %*
+java %KC_OPTS% -cp "%DIRNAME%\client\keycloak-admin-cli-${project.version}.jar" -Dkc.lib.dir="%DIRNAME%\client\lib" org.keycloak.client.admin.cli.KcAdmMain %*

integration/client-cli/admin-cli/src/main/bin/kcadm.sh

@@ -29,4 +29,4 @@ if [ "x$JAVA" = "x" ]; then
     fi
 fi
 
-"$JAVA" $KC_OPTS -cp $DIRNAME/client/keycloak-admin-cli-${project.version}.jar org.keycloak.client.admin.cli.KcAdmMain "$@"
+"$JAVA" $KC_OPTS -cp $DIRNAME/client/keycloak-admin-cli-${project.version}.jar -Dkc.lib.dir=$DIRNAME/client/lib org.keycloak.client.admin.cli.KcAdmMain "$@"

integration/client-cli/admin-cli/src/main/java/org/keycloak/client/admin/cli/KcAdmMain.java

@@ -27,6 +27,7 @@ import org.keycloak.client.admin.cli.aesh.AeshEnhancer;
 import org.keycloak.client.admin.cli.aesh.Globals;
 import org.keycloak.client.admin.cli.aesh.ValveInputStream;
 import org.keycloak.client.admin.cli.commands.KcAdmCmd;
+import org.keycloak.client.admin.cli.util.ClassLoaderUtil;
 import org.keycloak.common.crypto.CryptoIntegration;
 
 import java.util.ArrayList;
@@ -38,8 +39,14 @@ import java.util.Arrays;
 public class KcAdmMain {
 
     public static void main(String [] args) {
+        String libDir = System.getProperty("kc.lib.dir");
+        if (libDir == null) {
+            throw new RuntimeException("System property kc.lib.dir needs to be set");
+        }
+        ClassLoader cl = ClassLoaderUtil.resolveClassLoader(libDir);
+        Thread.currentThread().setContextClassLoader(cl);
 
-        CryptoIntegration.init(KcAdmMain.class.getClassLoader());
+        CryptoIntegration.init(cl);
         
         Globals.stdin = new ValveInputStream();
 

integration/client-cli/admin-cli/src/main/java/org/keycloak/client/admin/cli/util/ClassLoaderUtil.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2022 Red Hat, Inc. and/or its affiliates
+ *  and other contributors as indicated by the @author tags.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.keycloak.client.admin.cli.util;
+
+import java.io.File;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.stream.Stream;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class ClassLoaderUtil {
+
+    /**
+     * Detect if BC FIPS jars are present in the given directory. Return classloader with appropriate JARS based on that
+     */
+    public static ClassLoader resolveClassLoader(String libDir) {
+        File[] jarsInDir = new File(libDir).listFiles(file -> file.getName().endsWith(".jar"));
+
+        // Detect if BC FIPS jars are present in the "client/lib" directory
+        boolean bcFipsJarPresent = Stream.of(jarsInDir).anyMatch(file -> file.getName().startsWith("bc-fips"));
+        String[] validJarPrefixes = bcFipsJarPresent ?  new String[] {"keycloak-crypto-fips1402", "bc-fips", "bctls-fips"} : new String[] {"keycloak-crypto-default", "bcprov-jdk15on"};
+        URL[] usedJars = Stream.of(jarsInDir)
+                .filter(file -> {
+                    for (String prefix : validJarPrefixes) {
+                        if (file.getName().startsWith(prefix + "-")) return true;
+                    }
+                    return false;
+                })
+                .map(file -> {
+                    try {
+                        return file.toURI().toURL();
+                    } catch (MalformedURLException ex) {
+                        throw new IllegalStateException("Error when converting file into URL. Please check the files in the directory " + jarsInDir, ex);
+                    }
+                }).toArray(URL[]::new);
+
+        return new URLClassLoader(usedJars, ClassLoaderUtil.class.getClassLoader());
+    }
+
+}

integration/client-cli/client-cli-dist/assembly.xml

@@ -56,6 +56,14 @@
             </includes>
             <outputDirectory>keycloak-client-tools/bin/client</outputDirectory>
         </dependencySet>
+        <dependencySet>
+            <includes>
+                <include>org.keycloak:keycloak-crypto-default</include>
+                <include>org.keycloak:keycloak-crypto-fips1402</include>
+                <include>org.bouncycastle:bcprov-jdk15on</include>
+            </includes>
+            <outputDirectory>keycloak-client-tools/bin/client/lib</outputDirectory>
+        </dependencySet>
     </dependencySets>
 
 </assembly>

integration/client-cli/client-cli-dist/pom.xml

@@ -38,6 +38,36 @@
             <groupId>org.keycloak</groupId>
             <artifactId>keycloak-admin-cli</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.keycloak</groupId>
+            <artifactId>keycloak-crypto-default</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>*</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.keycloak</groupId>
+            <artifactId>keycloak-crypto-fips1402</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>*</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.bouncycastle</groupId>
+            <artifactId>bcprov-jdk15on</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>*</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
     </dependencies>
 
     <build>

integration/client-cli/client-registration-cli/pom.xml

@@ -38,10 +38,6 @@
             <groupId>org.keycloak</groupId>
             <artifactId>keycloak-core</artifactId>
         </dependency>
-        <dependency>
-            <groupId>org.keycloak</groupId>
-            <artifactId>${keycloak.crypto.artifactId}</artifactId>
-        </dependency>
         <dependency>
             <groupId>org.jboss.logging</groupId>
             <artifactId>jboss-logging</artifactId>
@@ -95,18 +91,6 @@
                                         <include>org/keycloak/common/crypto/**</include>
                                     </includes>
                                 </filter>
-                                <filter>
-                                    <artifact>org.bouncycastle:bcprov-jdk15on</artifact>
-                                    <includes>
-                                        <include>**/**</include>
-                                    </includes>
-                                </filter>
-                                <filter>
-                                    <artifact>org.bouncycastle:bcpkix-jdk15on</artifact>
-                                    <excludes>
-                                        <exclude>**/**</exclude>
-                                    </excludes>
-                                </filter>
                                 <filter>
                                     <artifact>com.fasterxml.jackson.core:jackson-core</artifact>
                                     <includes>

integration/client-cli/client-registration-cli/src/main/bin/kcreg.bat

@@ -5,4 +5,4 @@ if "%OS%" == "Windows_NT" (
 ) else (
   set DIRNAME=.\
 )
-java %KC_OPTS% -cp "%DIRNAME%\client\keycloak-client-registration-cli-${project.version}.jar" org.keycloak.client.registration.cli.KcRegMain %*
+java %KC_OPTS% -cp "%DIRNAME%\client\keycloak-client-registration-cli-${project.version}.jar" -Dkc.lib.dir="%DIRNAME%\client\lib" org.keycloak.client.registration.cli.KcRegMain %*

integration/client-cli/client-registration-cli/src/main/bin/kcreg.sh

@@ -28,4 +28,4 @@ if [ "x$JAVA" = "x" ]; then
 fi
 
 DIRNAME=`dirname "$RESOLVED_NAME"`
-"$JAVA" $KC_OPTS -cp $DIRNAME/client/keycloak-client-registration-cli-${project.version}.jar org.keycloak.client.registration.cli.KcRegMain "$@"
+"$JAVA" $KC_OPTS -cp $DIRNAME/client/keycloak-client-registration-cli-${project.version}.jar -Dkc.lib.dir=$DIRNAME/client/lib org.keycloak.client.registration.cli.KcRegMain "$@"

integration/client-cli/client-registration-cli/src/main/java/org/keycloak/client/registration/cli/KcRegMain.java

@@ -11,6 +11,7 @@ import org.keycloak.client.registration.cli.aesh.AeshEnhancer;
 import org.keycloak.client.registration.cli.aesh.ValveInputStream;
 import org.keycloak.client.registration.cli.aesh.Globals;
 import org.keycloak.client.registration.cli.commands.KcRegCmd;
+import org.keycloak.client.registration.cli.util.ClassLoaderUtil;
 import org.keycloak.common.crypto.CryptoIntegration;
 
 import java.util.ArrayList;
@@ -22,8 +23,14 @@ import java.util.Arrays;
 public class KcRegMain {
 
     public static void main(String [] args) {
+        String libDir = System.getProperty("kc.lib.dir");
+        if (libDir == null) {
+            throw new RuntimeException("System property kc.lib.dir needs to be set");
+        }
+        ClassLoader cl = ClassLoaderUtil.resolveClassLoader(libDir);
+        Thread.currentThread().setContextClassLoader(cl);
 
-        CryptoIntegration.init(KcRegMain.class.getClassLoader());
+        CryptoIntegration.init(cl);
 
         Globals.stdin = new ValveInputStream();
 

integration/client-cli/client-registration-cli/src/main/java/org/keycloak/client/registration/cli/util/ClassLoaderUtil.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2022 Red Hat, Inc. and/or its affiliates
+ *  and other contributors as indicated by the @author tags.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.keycloak.client.registration.cli.util;
+
+import java.io.File;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.stream.Stream;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class ClassLoaderUtil {
+
+    /**
+     * Detect if BC FIPS jars are present in the given directory. Return classloader with appropriate JARS based on that
+     */
+    public static ClassLoader resolveClassLoader(String libDir) {
+        File[] jarsInDir = new File(libDir).listFiles(file -> file.getName().endsWith(".jar"));
+
+        // Detect if BC FIPS jars are present in the "client/lib" directory
+        boolean bcFipsJarPresent = Stream.of(jarsInDir).anyMatch(file -> file.getName().startsWith("bc-fips"));
+        String[] validJarPrefixes = bcFipsJarPresent ?  new String[] {"keycloak-crypto-fips1402", "bc-fips", "bctls-fips"} : new String[] {"keycloak-crypto-default", "bcprov-jdk15on"};
+        URL[] usedJars = Stream.of(jarsInDir)
+                .filter(file -> {
+                    for (String prefix : validJarPrefixes) {
+                        if (file.getName().startsWith(prefix + "-")) return true;
+                    }
+                    return false;
+                })
+                .map(file -> {
+                    try {
+                        return file.toURI().toURL();
+                    } catch (MalformedURLException ex) {
+                        throw new IllegalStateException("Error when converting file into URL. Please check the files in the directory " + jarsInDir, ex);
+                    }
+                }).toArray(URL[]::new);
+
+        return new URLClassLoader(usedJars, ClassLoaderUtil.class.getClassLoader());
+    }
+
+}

testsuite/integration-arquillian/servers/auth-server/common/fips/kc.java.security

@@ -18,6 +18,9 @@ security.provider.3=
 fips.provider.1=org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider
 fips.provider.2=org.bouncycastle.jsse.provider.BouncyCastleJsseProvider fips:BCFIPS
 fips.provider.3=
+#fips.provider.3=SunJGSS
+#fips.provider.4=XMLDSig
+#fips.provider.5=
 
 # Commented this provider for now (and also other providers) as it uses lots of non-FIPS services.
 # See https://access.redhat.com/documentation/en-us/openjdk/11/html-single/configuring_openjdk_11_on_rhel_with_fips/index#ref_openjdk-default-fips-configuration_openjdk

testsuite/integration-arquillian/servers/auth-server/quarkus/pom.xml

@@ -356,6 +356,17 @@
                                     <includeArtifactIds>bc-fips,bctls-fips,bcpkix-fips</includeArtifactIds>
                                 </configuration>
                             </execution>
+                            <execution>
+                                <id>copy-bcfips-deps-client</id>
+                                <phase>generate-resources</phase>
+                                <goals>
+                                    <goal>copy-dependencies</goal>
+                                </goals>
+                                <configuration>
+                                    <outputDirectory>${auth.server.home}/bin/client/lib</outputDirectory>
+                                    <includeArtifactIds>bc-fips,bctls-fips</includeArtifactIds>
+                                </configuration>
+                            </execution>
                         </executions>
                     </plugin>
 

testsuite/integration-arquillian/tests/base/pom.xml

@@ -865,6 +865,40 @@
                 </plugins>
             </build>
         </profile>
+
+        <profile>
+            <id>auth-server-fips140-2</id>
+
+            <build>
+                <plugins>
+                    <plugin>
+                        <artifactId>maven-resources-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <id>copy-bcfips-deps-client-tools</id>
+                                <phase>process-test-resources</phase>
+                                <goals>
+                                    <goal>copy-resources</goal>
+                                </goals>
+                                <configuration>
+                                    <outputDirectory>${containers.home}/keycloak-client-tools/bin/client/lib</outputDirectory>
+                                    <resources>
+                                        <resource>
+                                            <directory>${containers.home}/auth-server-quarkus/bin/client/lib</directory>
+                                            <includes>
+                                                <include>bc-fips-*</include>
+                                                <include>bctls-fips-*</include>
+                                            </includes>
+                                        </resource>
+                                    </resources>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+
     </profiles>
 
 </project>

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/AuthServerTestEnricher.java

@@ -40,6 +40,7 @@ import org.jboss.arquillian.test.spi.event.suite.BeforeClass;
 import org.jboss.arquillian.test.spi.event.suite.BeforeSuite;
 import org.jboss.logging.Logger;
 import org.keycloak.admin.client.Keycloak;
+import org.keycloak.common.crypto.FipsMode;
 import org.keycloak.common.util.StringPropertyReplacer;
 import org.keycloak.representations.idm.RealmRepresentation;
 import org.keycloak.services.error.KeycloakErrorHandler;
@@ -128,6 +129,10 @@ public class AuthServerTestEnricher {
 
     public static final String AUTH_SERVER_HOME_PROPERTY = "auth.server.home";
 
+    public static final String AUTH_SERVER_FIPS_MODE_PROPERTY = "auth.server.fips.mode";
+
+    public static final FipsMode AUTH_SERVER_FIPS_MODE = FipsMode.valueOf(System.getProperty(AUTH_SERVER_FIPS_MODE_PROPERTY, FipsMode.disabled.toString()));
+
     public static final String CACHE_SERVER_LIFECYCLE_SKIP_PROPERTY = "cache.server.lifecycle.skip";
     public static final boolean CACHE_SERVER_LIFECYCLE_SKIP = Boolean.parseBoolean(System.getProperty(CACHE_SERVER_LIFECYCLE_SKIP_PROPERTY, "false"));
 

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cli/AbstractCliTest.java

@@ -1,8 +1,10 @@
 package org.keycloak.testsuite.cli;
 
 import org.junit.Assert;
+import org.keycloak.common.crypto.FipsMode;
 import org.keycloak.representations.idm.RealmRepresentation;
 import org.keycloak.testsuite.AbstractKeycloakTest;
+import org.keycloak.testsuite.arquillian.AuthServerTestEnricher;
 import org.keycloak.testsuite.cli.exec.AbstractExec;
 
 import java.io.File;
@@ -48,7 +50,9 @@ public abstract class AbstractCliTest extends AbstractKeycloakTest {
                 throw new AssertionError("STDOUT: " + exe.stdoutString(), e);
             }
         }
-        if (stdErrLineCount != -1) {
+        // There is additional logging in case that BC FIPS libraries are used, so the count of logged lines don't match with the case with plain BC used
+        // Hence we test count of lines just with FIPS disabled
+        if (stdErrLineCount != -1 && isFipsDisabled()) {
             try {
                 assertLineCount("stderr output", exe.stderrLines(), stdErrLineCount);
             } catch (Throwable e) {
@@ -70,4 +74,8 @@ public abstract class AbstractCliTest extends AbstractKeycloakTest {
         Assert.assertTrue(label + " has " + lines.size() + " lines (expected: " + count + ")", lines.size() == count);
     }
 
+    private boolean isFipsDisabled() {
+        return AuthServerTestEnricher.AUTH_SERVER_FIPS_MODE == FipsMode.disabled;
+    }
+
 }

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cli/admin/AbstractAdmCliTest.java

@@ -254,7 +254,7 @@ public abstract class AbstractAdmCliTest extends AbstractCliTest {
                 " --realm test " + credentials + " " + extraOptions + " -s clientId=test-client -o");
 
         Assert.assertEquals("exitCode == 0", 0, exe.exitCode());
-        Assert.assertEquals("login message", loginMessage, exe.stderrLines().get(0));
+        Assert.assertTrue("login message expected. But the messages are: " + exe.stderrLines(), exe.stderrLines().stream().anyMatch(message -> message.equals(loginMessage)));
 
         ClientRepresentation client = JsonSerialization.readValue(exe.stdout(), ClientRepresentation.class);
         Assert.assertEquals("clientId", "test-client", client.getClientId());
@@ -309,7 +309,7 @@ public abstract class AbstractAdmCliTest extends AbstractCliTest {
 
         assertExitCodeAndStreamSizes(exe, 1, 0, 2 - linecountOffset);
         String resourceUri = serverUrl + "/admin/realms/test/clients/" + client.getId();
-        Assert.assertEquals("error message", "Resource not found for url: " + resourceUri, exe.stderrLines().get(1 - linecountOffset));
+        Assert.assertEquals("error message", "Resource not found for url: " + resourceUri, exe.stderrLines().get(exe.stderrLines().size() - 1));
 
         lastModified2 = configFile.exists() ? configFile.lastModified() : 0;
         Assert.assertEquals("config file not modified", lastModified, lastModified2);

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cli/admin/KcAdmCreateTest.java

@@ -145,7 +145,7 @@ public class KcAdmCreateTest extends AbstractAdmCliTest {
             exe = execute("create clients --config '" + configFile.getName() + "' -s clientId=my_client4");
 
             assertExitCodeAndStreamSizes(exe, 0, 0, 1);
-            Assert.assertTrue("only id returned", exe.stderrLines().get(0).startsWith("Created new client with id '"));
+            Assert.assertTrue("only id returned", exe.stderrLines().get(exe.stderrLines().size() - 1).startsWith("Created new client with id '"));
         }
     }
 }

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cli/admin/KcAdmSessionTest.java

@@ -39,7 +39,7 @@ public class KcAdmSessionTest extends AbstractAdmCliTest {
             KcAdmExec exe = execute("create realms --config '" + configFile.getName() + "' -s realm=demorealm -s enabled=true");
 
             assertExitCodeAndStreamSizes(exe, 0, 0, 1);
-            Assert.assertTrue(exe.stderrLines().get(0).startsWith("Created "));
+            Assert.assertTrue(exe.stderrLines().get(exe.stderrLines().size() - 1).startsWith("Created "));
 
             // create user
             exe = execute("create users --config '" + configFile.getName() + "' -r demorealm -s username=testuser -s enabled=true -i");
@@ -95,7 +95,7 @@ public class KcAdmSessionTest extends AbstractAdmCliTest {
             exe = execute("create clients/" + idOfClient + "/roles --config '" + configFile.getName() + "' -s name=clientrole  -s 'description=Test client role'");
 
             assertExitCodeAndStreamSizes(exe, 0, 0, 1);
-            Assert.assertTrue(exe.stderrLines().get(0).startsWith("Created "));
+            Assert.assertTrue(exe.stderrLines().get(exe.stderrLines().size() - 1).startsWith("Created "));
 
             // make sure client role has been created
             exe = execute("get-roles --config '" + configFile.getName() + "' --cclientid testclient");

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cli/admin/KcAdmTest.java

@@ -474,7 +474,7 @@ public class KcAdmTest extends AbstractAdmCliTest {
 
         assertExitCodeAndStreamSizes(exe, 1, 0, 2);
         Assert.assertEquals("login message", "Logging into " + serverUrl + " as user user1 of realm test", exe.stderrLines().get(0));
-        Assert.assertEquals("error message", "Client not allowed for direct access grants [unauthorized_client]", exe.stderrLines().get(1));
+        Assert.assertEquals("error message", "Client not allowed for direct access grants [unauthorized_client]", exe.stderrLines().get(exe.stderrLines().size() - 1));
 
 
         // try wrong user password
@@ -483,7 +483,7 @@ public class KcAdmTest extends AbstractAdmCliTest {
 
         assertExitCodeAndStreamSizes(exe, 1, 0, 2);
         Assert.assertEquals("login message", "Logging into " + serverUrl + " as user user1 of realm test", exe.stderrLines().get(0));
-        Assert.assertEquals("error message", "Invalid user credentials [invalid_grant]", exe.stderrLines().get(1));
+        Assert.assertEquals("error message", "Invalid user credentials [invalid_grant]", exe.stderrLines().get(exe.stderrLines().size() - 1));
 
 
         // try wrong client secret
@@ -492,7 +492,7 @@ public class KcAdmTest extends AbstractAdmCliTest {
 
         assertExitCodeAndStreamSizes(exe, 1, 0, 2);
         Assert.assertEquals("login message", "Logging into " + serverUrl + " as user user1 of realm test", exe.stderrLines().get(0));
-        Assert.assertEquals("error message", "Invalid client or Invalid client credentials [unauthorized_client]", exe.stderrLines().get(1));
+        Assert.assertEquals("error message", "Invalid client or Invalid client credentials [unauthorized_client]", exe.stderrLines().get(exe.stderrLines().size() - 1));
 
 
         // try whole CRUD
@@ -516,7 +516,7 @@ public class KcAdmTest extends AbstractAdmCliTest {
 
         assertExitCodeAndStreamSizes(exe, 1, 0, 2);
         Assert.assertEquals("login message", "Logging into " + serverUrl + " as user user1 of realm test", exe.stderrLines().get(0));
-        Assert.assertEquals("error message", "Client not allowed for direct access grants [unauthorized_client]", exe.stderrLines().get(1));
+        Assert.assertEquals("error message", "Client not allowed for direct access grants [unauthorized_client]", exe.stderrLines().get(exe.stderrLines().size() - 1));
 
 
         // try wrong user password
@@ -526,7 +526,7 @@ public class KcAdmTest extends AbstractAdmCliTest {
 
         assertExitCodeAndStreamSizes(exe, 1, 0, 2);
         Assert.assertEquals("login message", "Logging into " + serverUrl + " as user user1 of realm test", exe.stderrLines().get(0));
-        Assert.assertEquals("error message", "Invalid user credentials [invalid_grant]", exe.stderrLines().get(1));
+        Assert.assertEquals("error message", "Invalid user credentials [invalid_grant]", exe.stderrLines().get(exe.stderrLines().size() - 1));
 
 
         // try wrong storepass
@@ -536,7 +536,7 @@ public class KcAdmTest extends AbstractAdmCliTest {
 
         assertExitCodeAndStreamSizes(exe, 1, 0, 2);
         Assert.assertEquals("login message", "Logging into " + serverUrl + " as user user1 of realm test", exe.stderrLines().get(0));
-        Assert.assertEquals("error message", "Failed to load private key: Keystore was tampered with, or password was incorrect", exe.stderrLines().get(1));
+        Assert.assertEquals("error message", "Failed to load private key: Keystore was tampered with, or password was incorrect", exe.stderrLines().get(exe.stderrLines().size() - 1));
 
 
         // try whole CRUD

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cli/registration/AbstractRegCliTest.java

@@ -487,7 +487,7 @@ public abstract class AbstractRegCliTest extends AbstractCliTest {
         exe = execute("delete test-client --no-config --server " + serverUrl + " --realm test " + credentials + " " + extraOptions);
 
         assertExitCodeAndStreamSizes(exe, 1, 0, 2);
-        Assert.assertEquals("error message", "Client not found [invalid_request]", exe.stderrLines().get(1));
+        Assert.assertEquals("error message", "Client not found [invalid_request]", exe.stderrLines().get(exe.stderrLines().size() - 1));
 
         lastModified2 = configFile.exists() ? configFile.lastModified() : 0;
         Assert.assertEquals("config file not modified", lastModified, lastModified2);

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cli/registration/KcRegCreateTest.java

@@ -13,6 +13,7 @@ import org.keycloak.client.registration.cli.config.ConfigData;
 import org.keycloak.client.registration.cli.config.FileConfigHandler;
 import org.keycloak.common.Profile;
 import org.keycloak.common.constants.ServiceAccountConstants;
+import org.keycloak.common.crypto.FipsMode;
 import org.keycloak.representations.idm.ClientRepresentation;
 import org.keycloak.representations.idm.RoleRepresentation;
 import org.keycloak.representations.idm.UserRepresentation;
@@ -20,6 +21,7 @@ import org.keycloak.representations.idm.authorization.PolicyEnforcementMode;
 import org.keycloak.representations.idm.authorization.ResourceServerRepresentation;
 import org.keycloak.representations.oidc.OIDCClientRepresentation;
 import org.keycloak.testsuite.ProfileAssume;
+import org.keycloak.testsuite.arquillian.AuthServerTestEnricher;
 import org.keycloak.testsuite.cli.KcRegExec;
 import org.keycloak.testsuite.util.TempFileResource;
 import org.keycloak.util.JsonSerialization;
@@ -171,7 +173,7 @@ public class KcRegCreateTest extends AbstractRegCliTest {
             exe = execute("create --insecure --config '" + configFile.getName() + "' -s clientId=my_client4");
 
             assertExitCodeAndStreamSizes(exe, 0, 0, 3);
-            Assert.assertEquals("only clientId returned", "Registered new client with client_id 'my_client4'", exe.stderrLines().get(2));
+            Assert.assertEquals("only clientId returned", "Registered new client with client_id 'my_client4'", exe.stderrLines().get(exe.stderrLines().size() - 1));
 
 
 
@@ -210,6 +212,8 @@ public class KcRegCreateTest extends AbstractRegCliTest {
                 Assert.assertEquals("Error message", "Attribute 'redirect_uris' not supported on document type 'default'", exe.stderrLines().get(0));
             }
 
+            // TODO: SAML is not tested with FIPS enabled as it does not work. This needs to be revisited when SAML works with FIPS
+            if (AuthServerTestEnricher.AUTH_SERVER_FIPS_MODE == FipsMode.disabled) {
 
                 // test create saml formated xml - format autodetection
                 File samlSpMetaFile = new File(System.getProperty("user.dir") + "/src/test/resources/cli/kcreg/saml-sp-metadata.xml");
@@ -226,7 +230,7 @@ public class KcRegCreateTest extends AbstractRegCliTest {
                 Assert.assertEquals("attributes.saml_name_id_format", "username", client.getAttributes().get("saml_name_id_format"));
                 Assert.assertEquals("attributes.saml_assertion_consumer_url_post", "http://localhost:8081/sales-post-enc/saml", client.getAttributes().get("saml_assertion_consumer_url_post"));
                 Assert.assertEquals("attributes.saml.signature.algorithm", "RSA_SHA256", client.getAttributes().get("saml.signature.algorithm"));
-
+            }
 
             // delete initial token
             exe = execute("config initial-token --config '" + configFile.getName() + "' --insecure --server " + serverUrl + " --realm " + realm + " --delete");

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cli/registration/KcRegTest.java

@@ -475,7 +475,7 @@ public class KcRegTest extends AbstractRegCliTest {
 
         assertExitCodeAndStreamSizes(exe, 1, 0, 2);
         Assert.assertEquals("login message", "Logging into " + serverUrl + " as user user1 of realm test", exe.stderrLines().get(0));
-        Assert.assertEquals("error message", "Client not allowed for direct access grants [unauthorized_client]", exe.stderrLines().get(1));
+        Assert.assertEquals("error message", "Client not allowed for direct access grants [unauthorized_client]", exe.stderrLines().get(exe.stderrLines().size() - 1));
 
 
         // try wrong user password
@@ -484,7 +484,7 @@ public class KcRegTest extends AbstractRegCliTest {
 
         assertExitCodeAndStreamSizes(exe, 1, 0, 2);
         Assert.assertEquals("login message", "Logging into " + serverUrl + " as user user1 of realm test", exe.stderrLines().get(0));
-        Assert.assertEquals("error message", "Invalid user credentials [invalid_grant]", exe.stderrLines().get(1));
+        Assert.assertEquals("error message", "Invalid user credentials [invalid_grant]", exe.stderrLines().get(exe.stderrLines().size() - 1));
 
 
         // try wrong client secret
@@ -493,7 +493,7 @@ public class KcRegTest extends AbstractRegCliTest {
 
         assertExitCodeAndStreamSizes(exe, 1, 0, 2);
         Assert.assertEquals("login message", "Logging into " + serverUrl + " as user user1 of realm test", exe.stderrLines().get(0));
-        Assert.assertEquals("error message", "Invalid client or Invalid client credentials [unauthorized_client]", exe.stderrLines().get(1));
+        Assert.assertEquals("error message", "Invalid client or Invalid client credentials [unauthorized_client]", exe.stderrLines().get(exe.stderrLines().size() - 1));
 
 
         // try whole CRUD
@@ -517,7 +517,7 @@ public class KcRegTest extends AbstractRegCliTest {
 
         assertExitCodeAndStreamSizes(exe, 1, 0, 2);
         Assert.assertEquals("login message", "Logging into " + serverUrl + " as user user1 of realm test", exe.stderrLines().get(0));
-        Assert.assertEquals("error message", "Client not allowed for direct access grants [unauthorized_client]", exe.stderrLines().get(1));
+        Assert.assertEquals("error message", "Client not allowed for direct access grants [unauthorized_client]", exe.stderrLines().get(exe.stderrLines().size() - 1));
 
 
         // try wrong user password
@@ -527,7 +527,7 @@ public class KcRegTest extends AbstractRegCliTest {
 
         assertExitCodeAndStreamSizes(exe, 1, 0, 2);
         Assert.assertEquals("login message", "Logging into " + serverUrl + " as user user1 of realm test", exe.stderrLines().get(0));
-        Assert.assertEquals("error message", "Invalid user credentials [invalid_grant]", exe.stderrLines().get(1));
+        Assert.assertEquals("error message", "Invalid user credentials [invalid_grant]", exe.stderrLines().get(exe.stderrLines().size() - 1));
 
 
         // try wrong storepass
@@ -537,7 +537,7 @@ public class KcRegTest extends AbstractRegCliTest {
 
         assertExitCodeAndStreamSizes(exe, 1, 0, 2);
         Assert.assertEquals("login message", "Logging into " + serverUrl + " as user user1 of realm test", exe.stderrLines().get(0));
-        Assert.assertEquals("error message", "Failed to load private key: Keystore was tampered with, or password was incorrect", exe.stderrLines().get(1));
+        Assert.assertEquals("error message", "Failed to load private key: Keystore was tampered with, or password was incorrect", exe.stderrLines().get(exe.stderrLines().size() - 1));
 
 
         // try whole CRUD

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cli/registration/KcRegUpdateTest.java

@@ -100,7 +100,7 @@ public class KcRegUpdateTest extends AbstractRegCliTest {
             exe = execute("update my_client --config '" + configFile.getName() + "' -o -s enabled=true -e oidc");
 
             assertExitCodeAndStreamSizes(exe, 1, 0, 1);
-            Assert.assertEquals("error message", "Failed to set attribute 'enabled' on document type 'oidc'", exe.stderrLines().get(0));
+            Assert.assertEquals("error message", "Failed to set attribute 'enabled' on document type 'oidc'", exe.stderrLines().get(exe.stderrLines().size() - 1));
 
 
 

testsuite/integration-arquillian/tests/pom.xml

@@ -1864,6 +1864,12 @@
                 <dependency>
                     <groupId>org.keycloak</groupId>
                     <artifactId>keycloak-client-cli-dist</artifactId>
+                    <exclusions>
+                        <exclusion>
+                            <groupId>org.keycloak</groupId>
+                            <artifactId>keycloak-crypto-fips1402</artifactId>
+                        </exclusion>
+                    </exclusions>
                     <type>zip</type>
                 </dependency>