allows csv output to handle missing requested fields (#23459)

* allows csv output to handle missing requested fields

Closes #12330

* fixes the handling of the content type

also makes it more explicit the expectation of applying csv and return
fields

* fix: consolidating the logic dealing with the content-type

Closes #23580

integration/client-cli/admin-cli/src/main/java/org/keycloak/client/admin/cli/commands/AbstractRequestCmd.java

@@ -18,6 +18,8 @@ package org.keycloak.client.admin.cli.commands;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+
+import org.apache.http.entity.ContentType;
 import org.jboss.aesh.console.command.CommandException;
 import org.jboss.aesh.console.command.CommandResult;
 import org.jboss.aesh.console.command.invocation.CommandInvocation;
@@ -398,11 +400,10 @@ public abstract class AbstractRequestCmd extends AbstractAuthOptionsCmd {
                 }
             }
 
-            Header contentType = response.getHeaders().get("content-type");
+            boolean json = response.getHeaders().getContentType().map(ContentType::getMimeType)
-            boolean canPrettyPrint = contentType != null && contentType.getValue().equals("application/json");
+                    .filter("application/json"::equals).isPresent();
-            boolean pretty = !compressed;
 
-            if (canPrettyPrint && (pretty || returnFields != null)) {
+            if (json && !compressed) {
                 ByteArrayOutputStream buffer = new ByteArrayOutputStream();
                 copyStream(response.getBody(), buffer);
 
@@ -417,10 +418,16 @@ public abstract class AbstractRequestCmd extends AbstractAuthOptionsCmd {
                     } else {
                         printAsCsv(rootNode, returnFields, unquoted);
                     }
-                } catch (Exception ignored) {
+                } catch (Exception e) {
-                    copyStream(new ByteArrayInputStream(buffer.toByteArray()), abos);
+                    throw new RuntimeException("Error processing results: " + e.getMessage(), e);
                 }
             } else {
+                if (outputFormat != OutputFormat.JSON || returnFields != null) {
+                    printErr("Cannot create CSV nor filter returned fields because the response is " + (compressed ? "compressed":"not json"));
+                    return CommandResult.SUCCESS;
+                }
+                // in theory the user could explicitly request json, but this could be a non-json response
+                // since there's no option for raw and we don't differentiate the default, there's no error about this
                 copyStream(response.getBody(), abos);
             }
         }

integration/client-cli/admin-cli/src/main/java/org/keycloak/client/admin/cli/util/Headers.java

@@ -16,8 +16,11 @@
  */
 package org.keycloak.client.admin.cli.util;
 
+import org.apache.http.entity.ContentType;
+
 import java.util.Iterator;
 import java.util.LinkedHashMap;
+import java.util.Optional;
 
 /**
  * @author <a href="mailto:mstrukel@redhat.com">Marko Strukelj</a>
@@ -52,4 +55,8 @@ public class Headers implements Iterable<Header> {
     public Iterator<Header> iterator() {
         return headers.values().iterator();
     }
+
+    public Optional<ContentType> getContentType() {
+        return Optional.ofNullable(headers.get("content-type")).map(Header::getValue).map(ContentType::parse);
+    }
 }

integration/client-cli/admin-cli/src/main/java/org/keycloak/client/admin/cli/util/HeadersBody.java

@@ -16,6 +16,8 @@
  */
 package org.keycloak.client.admin.cli.util;
 
+import org.apache.http.entity.ContentType;
+
 import java.io.ByteArrayOutputStream;
 import java.io.InputStream;
 import java.nio.charset.Charset;
@@ -50,7 +52,7 @@ public class HeadersBody {
 
     public String readBodyString() {
         byte [] buffer = readBodyBytes();
-        return new String(buffer, Charset.forName(getContentCharset()));
+        return new String(buffer, getContentCharset());
     }
 
     public byte[] readBodyBytes() {
@@ -59,14 +61,8 @@ public class HeadersBody {
         return os.toByteArray();
     }
 
-    public String getContentCharset() {
+    public Charset getContentCharset() {
-        Header contentType = headers.get("Content-Type");
+        return headers.getContentType().map(ContentType::getCharset).orElseGet(() -> Charset.forName("iso-8859-1"));
-        if (contentType != null) {
-            int pos = contentType.getValue().lastIndexOf("charset=");
-            if (pos != -1) {
-                return contentType.getValue().substring(pos + 8);
-            }
-        }
-        return "iso-8859-1";
     }
+
 }

integration/client-cli/admin-cli/src/main/java/org/keycloak/client/admin/cli/util/OutputUtil.java

@@ -22,14 +22,11 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.TextNode;
-import org.keycloak.util.JsonSerialization;
 
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.Iterator;
 import java.util.Map;
-
+import java.util.function.Consumer;
-import static org.keycloak.client.admin.cli.util.IoUtil.printOut;
 
 /**
  * @author <a href="mailto:mstrukel@redhat.com">Marko Strukelj</a>
@@ -48,13 +45,14 @@ public class OutputUtil {
             return (JsonNode) object;
         }
 
-        ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+        return MAPPER.convertValue(object, JsonNode.class);
-        buffer.write(JsonSerialization.writeValueAsBytes(object));
-        return MAPPER.readValue(buffer.toByteArray(), JsonNode.class);
     }
 
-
     public static void printAsCsv(Object object, ReturnFields fields, boolean unquoted) throws IOException {
+        printAsCsv(object, fields, unquoted, IoUtil::printOut);
+    }
+
+    public static void printAsCsv(Object object, ReturnFields fields, boolean unquoted, Consumer<String> printer) throws IOException {
 
         JsonNode node = convertToJsonNode(object);
         if (!node.isArray()) {
@@ -67,7 +65,7 @@ public class OutputUtil {
             StringBuilder buffer = new StringBuilder();
             printObjectAsCsv(buffer, item, fields, unquoted);
 
-            printOut(buffer.length() > 0 ? buffer.substring(1) : "");
+            printer.accept(buffer.length() > 0 ? buffer.substring(1) : "");
         }
     }
 
@@ -77,7 +75,9 @@ public class OutputUtil {
 
     static void printObjectAsCsv(StringBuilder out, JsonNode node, ReturnFields fields, boolean unquoted) {
 
-        if (node.isObject()) {
+        if (node == null) {
+            out.append(",");
+        } else if (node.isObject()) {
             if (fields == null) {
                 Iterator<Map.Entry<String, JsonNode>> it = node.fields();
                 while (it.hasNext()) {
@@ -95,7 +95,7 @@ public class OutputUtil {
             for (JsonNode item: node) {
                 printObjectAsCsv(out, item, fields, unquoted);
             }
-        } else if (node != null) {
+        } else {
             out.append(",");
             if (unquoted && node instanceof TextNode) {
                 out.append(node.asText());

integration/client-cli/admin-cli/src/test/java/org/keycloak/client/admin/cli/util/OuputUtilTest.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.client.admin.cli.util;
+
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import static org.junit.Assert.assertEquals;
+
+public class OuputUtilTest {
+
+    @Test
+    public void testConversionToCsv() throws IOException {
+        HashMap<Object, Object> map1 = new HashMap<>();
+        map1.put("not-x", "omit");
+        map1.put("y", "v1");
+        HashMap<Object, Object> map2 = new HashMap<>();
+        map2.put("x", "v2");
+        JsonNode node = OutputUtil.convertToJsonNode(Arrays.asList(map1, map2));
+        ArrayList<String> result = new ArrayList<>();
+        OutputUtil.printAsCsv(node, new ReturnFields("x,y"), false, result::add);
+        assertEquals(",\"v1\"", result.get(0));
+        assertEquals("\"v2\",", result.get(1));
+    }
+
+}

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cli/admin/KcAdmTest.java

@@ -610,4 +610,30 @@ public class KcAdmTest extends AbstractAdmCliTest {
         KcAdmExec exec = execute("add-roles --uusername=testuser --rolename offline_access --target-realm=demorealm");
         Assert.assertEquals(0, exec.exitCode());
     }
+
+    @Test
+    public void testCsvFormat() {
+        execute("config credentials --server " + serverUrl + " --realm master --user admin --password admin");
+        KcAdmExec exec = execute("get realms/master --format csv");
+        assertExitCodeAndStreamSizes(exec, 0, 1, 0);
+        Assert.assertTrue(exec.stdoutString().startsWith("\""));
+    }
+
+    @Test
+    public void testCsvFormatWithMissingFields() {
+        execute("config credentials --server " + serverUrl + " --realm master --user admin --password admin");
+        KcAdmExec exec = execute("get realms/master --format csv --fields foo");
+        // nothing valid was selected, should be blank
+        assertExitCodeAndStreamSizes(exec, 0, 1, 0);
+        Assert.assertTrue(exec.stdoutString().isBlank());
+    }
+
+    @Test
+    public void testCompressedCsv() {
+        execute("config credentials --server " + serverUrl + " --realm master --user admin --password admin");
+        KcAdmExec exec = execute("get realms/master --format csv --compressed");
+        // should contain an error message
+        assertExitCodeAndStreamSizes(exec, 0, 0, 1);
+    }
+
 }