KEYCLOAK-3789 KcRegTests are failing on CI

- Ignore occasional extra empty lines at end of stream

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cli/registration/AbstractCliTest.java

@@ -539,17 +539,30 @@ public abstract class AbstractCliTest extends AbstractKeycloakTest {
         Assert.assertEquals("exitCode == " + exitCode, exitCode, exe.exitCode());
         if (stdOutLineCount != -1) {
             try {
-                Assert.assertTrue("stdout output has " + stdOutLineCount + " lines", exe.stdoutLines().size() == stdOutLineCount);
+                assertLineCount("stdout output", exe.stdoutLines(), stdOutLineCount);
             } catch (Throwable e) {
                 throw new AssertionError("STDOUT: " + exe.stdoutString(), e);
             }
         }
         if (stdErrLineCount != -1) {
             try {
-                Assert.assertTrue("stderr output has " + stdErrLineCount + " lines", exe.stderrLines().size() == stdErrLineCount);
+                assertLineCount("stderr output", exe.stderrLines(), stdErrLineCount);
             } catch (Throwable e) {
                 throw new AssertionError("STDERR: " + exe.stderrString(), e);
             }
         }
     }
+
+    void assertLineCount(String label, List<String> lines, int count) {
+        if (lines.size() == count) {
+            return;
+        }
+        // there is some kind of race condition in 'kcreg' that results in intermittent extra empty line
+        if (lines.size() == count + 1) {
+            if ("".equals(lines.get(lines.size()-1))) {
+                return;
+            }
+        }
+        Assert.assertTrue(label + " has " + lines.size() + " lines (expected: " + count + ")", lines.size() == count);
+    }
 }

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cli/registration/KcRegCreateTest.java

@@ -32,13 +32,13 @@ public class KcRegCreateTest extends AbstractCliTest {
             KcRegExec exe = execute("config credentials -x --config '" + configFile.getName() +
                     "' --server " + serverUrl + " --realm master --user admin --password admin");
 
-            Assert.assertEquals("exitCode == 0", 0, exe.exitCode());
+            assertExitCodeAndStreamSizes(exe, 0, 0, 1);
 
             // use initial token of another realm with server, and realm override
             String token = issueInitialAccessToken("test");
             exe = execute("create --config '" + configFile.getName() + "' --server " + serverUrl + " --realm test -s clientId=my_first_client -t " + token);
 
-            Assert.assertEquals("exitCode == 0", 0, exe.exitCode());
+            assertExitCodeAndStreamSizes(exe, 0, 0, 1);
         }
     }
 
@@ -57,7 +57,7 @@ public class KcRegCreateTest extends AbstractCliTest {
             KcRegExec exe = execute("config initial-token -x --config '" + configFile.getName() +
                     "' --server " + serverUrl + " --realm " + realm + " " + token);
 
-            Assert.assertEquals("exitCode == 0", 0, exe.exitCode());
+            assertExitCodeAndStreamSizes(exe, 0, 0, 0);
 
             // check that current server, realm, and initial token are saved in the file
             ConfigData config = handler.loadConfig();
@@ -87,8 +87,7 @@ public class KcRegCreateTest extends AbstractCliTest {
 
                 exe = execute("create --config '" + configFile.getName() + "' -o -f - < '" + tmpFile.getName() + "'");
 
-                Assert.assertEquals("exitCode == 0", 0, exe.exitCode());
+                assertExitCodeAndStdErrSize(exe, 0, 0);
-                Assert.assertTrue("stderr is empty", exe.stderrLines().isEmpty());
 
                 ClientRepresentation client = JsonSerialization.readValue(exe.stdout(), ClientRepresentation.class);
                 Assert.assertNotNull("id", client.getId());
@@ -114,8 +113,7 @@ public class KcRegCreateTest extends AbstractCliTest {
                         " -s 'name=My Client App II' -s protocol=keycloak-oidc -s 'webOrigins=[\"http://localhost:8980/myapp2\"]'" +
                         " -s baseUrl=http://localhost:8980/myapp2 -s rootUrl=http://localhost:8980/myapp2");
 
-                Assert.assertEquals("exitCode == 0", 0, exe.exitCode());
+                assertExitCodeAndStdErrSize(exe, 0, 0);
-                Assert.assertTrue("stderr is empty", exe.stderrLines().isEmpty());
 
                 ClientRepresentation client2 = JsonSerialization.readValue(exe.stdout(), ClientRepresentation.class);
                 Assert.assertNotNull("id", client2.getId());
@@ -138,29 +136,22 @@ public class KcRegCreateTest extends AbstractCliTest {
                 // check that using an invalid attribute key is not ignored
                 exe = execute("create --config '" + configFile.getName() + "' -o -f '" + tmpFile.getName() + "' -s client_id=my_client3");
 
-                Assert.assertEquals("exitCode == 1", 1, exe.exitCode());
+                assertExitCodeAndStreamSizes(exe, 1, 0, 1);
-                Assert.assertEquals("stderr has one line", 1, exe.stderrLines().size());
                 Assert.assertEquals("Failed to set attribute 'client_id' on document type 'default'", exe.stderrLines().get(0));
             }
 
             // simple create, output an id
             exe = execute("create --config '" + configFile.getName() + "' -i -s clientId=my_client3");
 
-            Assert.assertEquals("exitCode == 0", 0, exe.exitCode());
+            assertExitCodeAndStreamSizes(exe, 0, 1, 0);
-            Assert.assertEquals("stderr is empty", 0, exe.stderrLines().size());
-
-            Assert.assertEquals("stdout has 1 line", 1, exe.stdoutLines().size());
             Assert.assertEquals("only clientId returned", "my_client3", exe.stdoutLines().get(0));
 
             // simple create, default output
             exe = execute("create --config '" + configFile.getName() + "' -s clientId=my_client4");
 
-            Assert.assertEquals("exitCode == 0", 0, exe.exitCode());
+            assertExitCodeAndStreamSizes(exe, 0, 0, 1);
-            Assert.assertEquals("stderr has 1 line", 1, exe.stderrLines().size());
             Assert.assertEquals("only clientId returned", "Registered new client with client_id 'my_client4'", exe.stderrLines().get(0));
 
-            Assert.assertEquals("stdout is empty", 0, exe.stdoutLines().size());
-
 
 
             // create using oidc endpoint - autodetect format
@@ -178,8 +169,7 @@ public class KcRegCreateTest extends AbstractCliTest {
                         " -s 'redirect_uris=[\"http://localhost:8980/myapp5/*\"]' -s client_uri=http://localhost:8980/myapp5" +
                         " -o -f - < '" + tmpFile.getName() + "'");
 
-                Assert.assertEquals("exitCode == 0", 0, exe.exitCode());
+                assertExitCodeAndStdErrSize(exe, 0, 0);
-                Assert.assertTrue("stderr is empty", exe.stderrLines().isEmpty());
 
                 OIDCClientRepresentation client = JsonSerialization.readValue(exe.stdout(), OIDCClientRepresentation.class);
 
@@ -195,10 +185,8 @@ public class KcRegCreateTest extends AbstractCliTest {
                 // try use incompatible endpoint override
                 exe = execute("create --config '" + configFile.getName() + "' -e default -f '" + tmpFile.getName() + "'");
 
-                Assert.assertEquals("exitCode == 1", 1, exe.exitCode());
+                assertExitCodeAndStreamSizes(exe, 1, 0, 1);
-                Assert.assertFalse("stderr not empty", exe.stderrLines().isEmpty());
                 Assert.assertEquals("Error message", "Attribute 'redirect_uris' not supported on document type 'default'", exe.stderrLines().get(0));
-
             }
 
 
@@ -208,8 +196,7 @@ public class KcRegCreateTest extends AbstractCliTest {
 
             exe = execute("create --config '" + configFile.getName() + "' -o -f - < '" + samlSpMetaFile.getAbsolutePath() + "'");
 
-            Assert.assertEquals("exitCode == 0", 0, exe.exitCode());
+            assertExitCodeAndStdErrSize(exe, 0, 0);
-            Assert.assertTrue("stderr is empty", exe.stderrLines().isEmpty());
 
             ClientRepresentation client = JsonSerialization.readValue(exe.stdout(), ClientRepresentation.class);
             Assert.assertNotNull("id", client.getId());

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cli/registration/KcRegTest.java

@@ -33,7 +33,7 @@ public class KcRegTest extends AbstractCliTest {
          */
         KcRegExec exe = execute("");
 
-        Assert.assertEquals("exitCode", 1, exe.exitCode());
+        assertExitCodeAndStdErrSize(exe, 1, 0);
 
         List<String> lines = exe.stdoutLines();
         Assert.assertTrue("stdout output not empty", lines.size() > 0);
@@ -41,8 +41,6 @@ public class KcRegTest extends AbstractCliTest {
         Assert.assertEquals("stdout one but last line", "Use '" + KcRegExec.CMD + " help <command>' for more information about a given command.", lines.get(lines.size() - 2));
         Assert.assertEquals("stdout last line", "", lines.get(lines.size() - 1));
 
-        lines = exe.stderrLines();
-        Assert.assertTrue("stderr output empty", lines.size() == 0);
 
         /*
          * Test commands without arguments
@@ -79,7 +77,7 @@ public class KcRegTest extends AbstractCliTest {
         Assert.assertEquals("error message", "CLIENT not specified", exe.stderrLines().get(0));
 
         exe = execute("help");
-        Assert.assertEquals("exitCode == 0", 0, exe.exitCode());
+        assertExitCodeAndStdErrSize(exe, 0, 0);
         lines = exe.stdoutLines();
         Assert.assertTrue("stdout output not empty", lines.size() > 0);
         Assert.assertEquals("stdout first line", "Keycloak Client Registration CLI", lines.get(0));
@@ -93,69 +91,57 @@ public class KcRegTest extends AbstractCliTest {
          *  Test --help for all commands
          */
         KcRegExec exe = execute("--help");
-        Assert.assertEquals("exit code", 0, exe.exitCode());
+        assertExitCodeAndStdErrSize(exe, 0, 0);
-        Assert.assertEquals("no stderr", 0, exe.stderrLines().size());
         Assert.assertEquals("stdout first line", "Keycloak Client Registration CLI", exe.stdoutLines().get(0));
 
         exe = execute("create --help");
-        Assert.assertEquals("exit code", 0, exe.exitCode());
+        assertExitCodeAndStdErrSize(exe, 0, 0);
-        Assert.assertEquals("no stderr", 0, exe.stderrLines().size());
         Assert.assertEquals("stdout first line", "Usage: " + CMD + " create [ARGUMENTS]", exe.stdoutLines().get(0));
 
         exe = execute("get --help");
-        Assert.assertEquals("exit code", 0, exe.exitCode());
+        assertExitCodeAndStdErrSize(exe, 0, 0);
-        Assert.assertEquals("no stderr", 0, exe.stderrLines().size());
         Assert.assertEquals("stdout first line", "Usage: " + CMD + " get CLIENT [ARGUMENTS]", exe.stdoutLines().get(0));
 
         exe = execute("update --help");
-        Assert.assertEquals("exit code", 0, exe.exitCode());
+        assertExitCodeAndStdErrSize(exe, 0, 0);
-        Assert.assertEquals("no stderr", 0, exe.stderrLines().size());
         Assert.assertEquals("stdout first line", "Usage: " + CMD + " update CLIENT [ARGUMENTS]", exe.stdoutLines().get(0));
 
         exe = execute("delete --help");
-        Assert.assertEquals("exit code", 0, exe.exitCode());
+        assertExitCodeAndStdErrSize(exe, 0, 0);
-        Assert.assertEquals("no stderr", 0, exe.stderrLines().size());
         Assert.assertEquals("stdout first line", "Usage: " + CMD + " delete CLIENT [ARGUMENTS]", exe.stdoutLines().get(0));
 
         exe = execute("attrs --help");
-        Assert.assertEquals("exit code", 0, exe.exitCode());
+        assertExitCodeAndStdErrSize(exe, 0, 0);
-        Assert.assertEquals("no stderr", 0, exe.stderrLines().size());
         Assert.assertEquals("stdout first line", "Usage: " + CMD + " attrs [ATTRIBUTE] [ARGUMENTS]", exe.stdoutLines().get(0));
 
         exe = execute("update-token --help");
-        Assert.assertEquals("exit code", 0, exe.exitCode());
+        assertExitCodeAndStdErrSize(exe, 0, 0);
-        Assert.assertEquals("no stderr", 0, exe.stderrLines().size());
         Assert.assertEquals("stdout first line", "Usage: " + CMD + " update-token CLIENT [ARGUMENTS]", exe.stdoutLines().get(0));
 
         exe = execute("config --help");
-        Assert.assertEquals("exit code", 0, exe.exitCode());
+        assertExitCodeAndStdErrSize(exe, 0, 0);
-        Assert.assertEquals("no stderr", 0, exe.stderrLines().size());
         Assert.assertEquals("stdout first line", "Usage: " + OsUtil.CMD + " config SUB_COMMAND [ARGUMENTS]", exe.stdoutLines().get(0));
 
         exe = execute("config credentials --help");
-        Assert.assertEquals("exit code", 0, exe.exitCode());
+        assertExitCodeAndStdErrSize(exe, 0, 0);
-        Assert.assertEquals("no stderr", 0, exe.stderrLines().size());
         Assert.assertEquals("stdout first line",
                 "Usage: " + CMD + " config credentials --server SERVER_URL --realm REALM [ARGUMENTS]",
                 exe.stdoutLines().get(0));
 
         exe = execute("config initial-token --help");
-        Assert.assertEquals("exit code", 0, exe.exitCode());
+        assertExitCodeAndStdErrSize(exe, 0, 0);
-        Assert.assertEquals("no stderr", 0, exe.stderrLines().size());
         Assert.assertEquals("stdout first line",
                 "Usage: " + CMD + " config initial-token --server SERVER --realm REALM [--delete | TOKEN] [ARGUMENTS]",
                 exe.stdoutLines().get(0));
 
         exe = execute("config registration-token --help");
-        Assert.assertEquals("exit code", 0, exe.exitCode());
+        assertExitCodeAndStdErrSize(exe, 0, 0);
-        Assert.assertEquals("no stderr", 0, exe.stderrLines().size());
         Assert.assertEquals("stdout first line",
                 "Usage: " + CMD + " config registration-token --server SERVER --realm REALM --client CLIENT [--delete | TOKEN] [ARGUMENTS]",
                 exe.stdoutLines().get(0));
 
         exe = execute("config truststore --help");
-        Assert.assertEquals("exit code", 0, exe.exitCode());
+        assertExitCodeAndStdErrSize(exe, 0, 0);
-        Assert.assertEquals("no stderr", 0, exe.stderrLines().size());
         Assert.assertEquals("stdout first line",
                 "Usage: " + CMD + " config truststore [TRUSTSTORE | --delete] [--trustpass PASSWOD] [ARGUMENTS]",
                 exe.stdoutLines().get(0));
@@ -379,7 +365,7 @@ public class KcRegTest extends AbstractCliTest {
             KcRegExec exe = execute("config credentials --server " + serverUrl +
                     " --realm master --user admin --password admin --config '" + configFile.getName() + "'");
 
-            Assert.assertEquals("exitCode == 0", 0, exe.exitCode());
+            assertExitCodeAndStreamSizes(exe, 0, 0, 1);
 
             // remember the state of config file
             ConfigData config1 = handler.loadConfig();
@@ -389,7 +375,7 @@ public class KcRegTest extends AbstractCliTest {
 
             exe = execute("create --config '" + configFile.getName() + "' -s clientId=test-client -o");
 
-            Assert.assertEquals("exitCode == 0", 0, exe.exitCode());
+            assertExitCodeAndStdErrSize(exe, 0, 0);
 
             // check changes to config file
             ConfigData config2 = handler.loadConfig();
@@ -619,12 +605,17 @@ public class KcRegTest extends AbstractCliTest {
             KcRegExec exe = execute("create " + (useConfig ? ("--config '" + configFile.getAbsolutePath()) + "'" : "--no-config")
                     + " --server " + serverUrl + " --realm " + realm + " -s clientId=test-client -o");
 
-            Assert.assertEquals("exitCode == 0", 0, exe.exitCode());
+            assertExitCodeAndStdErrSize(exe, 0, 0);
 
             ClientRepresentation client = JsonSerialization.readValue(exe.stdout(), ClientRepresentation.class);
 
             Assert.assertEquals("clientId", "test-client", client.getClientId());
             Assert.assertNotNull("registrationAccessToken", client.getRegistrationAccessToken());
+
+            exe = execute("delete test-client " + (useConfig ? ("--config '" + configFile.getAbsolutePath()) + "'" : "--no-config")
+                    + " --server " + serverUrl + " --realm " + realm + " -t " + client.getRegistrationAccessToken());
+
+            assertExitCodeAndStreamSizes(exe, 0, 0, 0);
         }
     }
 

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cli/registration/KcRegUpdateTest.java

@@ -35,7 +35,8 @@ public class KcRegUpdateTest extends AbstractCliTest {
             // create an object so we can update it
             KcRegExec exe = execute("create --config '" + configFile.getName() + "' -o -s clientId=my_client");
 
-            Assert.assertEquals("exitCode == 0", 0, exe.exitCode());
+            assertExitCodeAndStdErrSize(exe, 0, 0);
+
             ClientRepresentation client = JsonSerialization.readValue(exe.stdout(), ClientRepresentation.class);
 
             Assert.assertEquals("enabled", true, client.isEnabled());
@@ -48,8 +49,7 @@ public class KcRegUpdateTest extends AbstractCliTest {
             exe = execute("update my_client --config '" + configFile.getName() + "' -o " +
                         " -s enabled=false -s 'redirectUris=[\"http://localhost:8980/myapp/*\"]'");
 
-            Assert.assertEquals("exitCode == 0", 0, exe.exitCode());
+            assertExitCodeAndStdErrSize(exe, 0, 0);
-            Assert.assertTrue("stderr is empty", exe.stderrLines().isEmpty());
 
             client = JsonSerialization.readValue(exe.stdout(), ClientRepresentation.class);
             Assert.assertEquals("enabled", false, client.isEnabled());
@@ -60,8 +60,7 @@ public class KcRegUpdateTest extends AbstractCliTest {
             // Another merge update - test deleting an attribute, deleting a list item and adding a list item
             exe = execute("update my_client --config '" + configFile.getName() + "' -o -d redirectUris -s webOrigins+=http://localhost:8980/myapp -s webOrigins+=http://localhost:8981/myapp -d webOrigins[0]");
 
-            Assert.assertEquals("exitCode == 0", 0, exe.exitCode());
+            assertExitCodeAndStdErrSize(exe, 0, 0);
-            Assert.assertTrue("stderr is empty", exe.stderrLines().isEmpty());
 
             client = JsonSerialization.readValue(exe.stdout(), ClientRepresentation.class);
 
@@ -76,8 +75,7 @@ public class KcRegUpdateTest extends AbstractCliTest {
             exe = execute("update my_client --config '" + configFile.getName() + "' -o -s 'protocolMappers[0].config.\"id.token.claim\"=false' " +
                     "-s 'protocolMappers[4].config={\"single\": \"true\", \"attribute.nameformat\": \"Basic\", \"attribute.name\": \"Role\"}'");
 
-            Assert.assertEquals("exitCode == 0", 0, exe.exitCode());
+            assertExitCodeAndStdErrSize(exe, 0, 0);
-            Assert.assertTrue("stderr is empty", exe.stderrLines().isEmpty());
 
             client = JsonSerialization.readValue(exe.stdout(), ClientRepresentation.class);
             Assert.assertEquals("protocolMapper[0].config.\"id.token.claim\"", "false", client.getProtocolMappers().get(0).getConfig().get("id.token.claim"));
@@ -112,8 +110,7 @@ public class KcRegUpdateTest extends AbstractCliTest {
                     .stdin(new ByteArrayInputStream("{ \"enabled\": false }".getBytes()))
                     .execute();
 
-            Assert.assertEquals("exitCode == 0", 0, exe.exitCode());
+            assertExitCodeAndStdErrSize(exe, 0, 0);
-            Assert.assertTrue("stderr has error", exe.stderrLines().isEmpty());
 
             client = JsonSerialization.readValue(exe.stdout(), ClientRepresentation.class);
             // web origin is not sent to the server, thus it retains the current value
@@ -130,8 +127,7 @@ public class KcRegUpdateTest extends AbstractCliTest {
                     .stdin(new ByteArrayInputStream("{ \"webOrigins\": [\"http://localhost:8980/myapp\"] }".getBytes()))
                     .execute();
 
-            Assert.assertEquals("exitCode == 0", 0, exe.exitCode());
+            assertExitCodeAndStdErrSize(exe, 0, 0);
-            Assert.assertTrue("stderr has error", exe.stderrLines().isEmpty());
 
             client = JsonSerialization.readValue(exe.stdout(), ClientRepresentation.class);
             Assert.assertEquals("webOrigins", Arrays.asList("http://localhost:8980/myapp"), client.getWebOrigins());
@@ -144,8 +140,7 @@ public class KcRegUpdateTest extends AbstractCliTest {
             exe = execute("config registration-token --config '" + configFile.getName() + "' --server " + serverUrl +
                     " --realm " + realm + " --client my_client -d");
 
-            Assert.assertEquals("exitCode == 0", 0, exe.exitCode());
+            assertExitCodeAndStdErrSize(exe, 0, 0);
-            Assert.assertTrue("stderr is empty", exe.stderrLines().isEmpty());
 
             Assert.assertNull("my_client registration token", handler.loadConfig().ensureRealmConfigData(serverUrl, realm).getClients().get("my_client"));
         }

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cli/registration/KcRegUpdateTokenTest.java

@@ -13,7 +13,6 @@ import org.keycloak.util.JsonSerialization;
 
 import java.io.IOException;
 
-import static org.keycloak.client.registration.cli.util.OsUtil.EOL;
 import static org.keycloak.testsuite.cli.KcRegExec.execute;
 
 /**
@@ -59,7 +58,8 @@ public class KcRegUpdateTokenTest extends AbstractCliTest {
 
             // test that the token works
             exe = execute("get reg-cli-secret-direct --no-config --server " + serverUrl + " --realm test -t " + token);
-            Assert.assertEquals("exit code", 0, exe.exitCode());
+
+            assertExitCodeAndStdErrSize(exe, 0, 0);
 
             ClientRepresentation client = JsonSerialization.readValue(exe.stdout(), ClientRepresentation.class);
             Assert.assertEquals("client representation returned", "reg-cli-secret-direct", client.getClientId());