Partial import phase 2 (#702)

* Process the JSON and present user options

* Finish checkboxes.  Refactor.

* Add tests

* Refactor after rebase

* Add more test data for manual testing.

* Fix linting errors

* Put JsonFileUpload back the way it was.

* Clean up comments

* Update src/realm-settings/PartialImport.tsx

Remove comment

Co-authored-by: Jenny <32821331+jenny-s51@users.noreply.github.com>

Co-authored-by: Jenny <32821331+jenny-s51@users.noreply.github.com>

cypress/integration/partial-import-test-data/client-only.json

@@ -0,0 +1,13 @@
+{
+   "clients": [
+        {
+            "clientId": "customer-portal",
+            "enabled": true,
+            "adminUrl": "/customer-portal",
+            "baseUrl": "/customer-portal",
+            "redirectUris": [
+                "/customer-portal/*"
+            ],
+            "secret": "password"
+        }]
+}

cypress/integration/partial-import-test-data/multi-realm.json

@@ -0,0 +1,113 @@
+[
+  {
+    "id": "realm1",
+    "realm": "realm1",
+    "clients": [
+      {
+        "clientId": "customer-portal",
+        "enabled": true,
+        "adminUrl": "/customer-portal",
+        "baseUrl": "/customer-portal",
+        "redirectUris": ["/customer-portal/*"],
+        "secret": "password"
+      }
+    ],
+    "users": [
+      {
+        "username": "ssilvert",
+        "enabled": true,
+        "email": "ssilvert@redhat.com",
+        "firstName": "Stan",
+        "lastName": "Silvert",
+        "credentials": [{ "type": "password", "value": "password" }],
+        "realmRoles": ["user", "offline_access"],
+        "clientRoles": {
+          "account": ["manage-account"]
+        }
+      }
+    ],
+    "groups": [
+      {
+        "id": "7a20471c-c695-4778-adb0-4ee4ae88d198",
+        "name": "group1",
+        "path": "/group1",
+        "attributes": {
+          "foo": ["bar"]
+        },
+        "realmRoles": ["create-realm"],
+        "clientRoles": {},
+        "subGroups": []
+      }
+    ],
+    "identityProviders": [
+      {
+        "alias": "keycloak-oidc",
+        "internalId": "721b5dae-5b98-4284-bcd1-f872ce2ac174",
+        "providerId": "keycloak-oidc",
+        "enabled": true,
+        "updateProfileFirstLoginMode": "on",
+        "trustEmail": false,
+        "storeToken": false,
+        "addReadTokenRoleOnCreate": false,
+        "authenticateByDefault": false,
+        "config": {
+          "clientSecret": "foo",
+          "clientId": "foo",
+          "tokenUrl": "https://foo.bar",
+          "authorizationUrl": "https://foo.bar"
+        }
+      }
+    ],
+    "roles": {
+      "realm": [
+        {
+          "id": "9d2638c8-4c62-4c42-90ea-5f3c836d0cc8",
+          "name": "offline_access",
+          "scopeParamRequired": false,
+          "composite": false
+        },
+        {
+          "id": "9d2638c8-4c62-4c42-90ea-5f3c836d0cc8",
+          "name": "another",
+          "scopeParamRequired": false,
+          "composite": false
+        }
+      ],
+      "client": {
+        "database-service": [],
+        "admin-client": [],
+        "realm-management": [
+          {
+            "id": "3b939f75-d013-4096-8462-48aa39261293",
+            "name": "create-client",
+            "description": "${role_create-client}",
+            "scopeParamRequired": false,
+            "composite": false
+          }
+        ]
+      }
+    }
+  },
+  {
+    "id": "realm2",
+    "realm": "realm2",
+    "clients": [
+      {
+        "clientId": "customer-portal",
+        "enabled": true,
+        "adminUrl": "/customer-portal",
+        "baseUrl": "/customer-portal",
+        "redirectUris": ["/customer-portal/*"],
+        "secret": "password"
+      },
+      {
+        "clientId": "customer-portal",
+        "enabled": true,
+        "adminUrl": "/customer-portal",
+        "baseUrl": "/customer-portal",
+        "redirectUris": ["/customer-portal/*"],
+        "secret": "password"
+      }
+    ]
+  }
+]

cypress/integration/partial_import_test.spec.ts

@@ -1,33 +1,94 @@
+import CreateRealmPage from "../support/pages/admin_console/CreateRealmPage";
 import SidebarPage from "../support/pages/admin_console/SidebarPage";
 import LoginPage from "../support/pages/LoginPage";
 import PartialImportModal from "../support/pages/admin_console/configure/realm_settings/PartialImportModal";
 import RealmSettings from "../support/pages/admin_console/configure/realm_settings/RealmSettings";
 import { keycloakBefore } from "../support/util/keycloak_before";
+import AdminClient from "../support/util/AdminClient";
 
 describe("Partial import test", () => {
+  const TEST_REALM = "partial-import-test-realm";
   const loginPage = new LoginPage();
   const sidebarPage = new SidebarPage();
-  const partialImportModal = new PartialImportModal();
+  const createRealmPage = new CreateRealmPage();
+  const modal = new PartialImportModal();
   const realmSettings = new RealmSettings();
 
-  beforeEach(function () {
+  beforeEach(() => {
     keycloakBefore();
     loginPage.logIn();
+
+    // doing this from the UI has the added bonus of putting you in the test realm
+    sidebarPage.goToCreateRealm();
+    createRealmPage.fillRealmName(TEST_REALM).createRealm();
+
     sidebarPage.goToRealmSettings();
     realmSettings.clickActionMenu();
   });
 
-  it("Opens and closes partial import dialog", () => {
-    partialImportModal.open();
-    cy.getId("import-button").should("be.disabled");
-    cy.getId("cancel-button").click();
-    cy.getId("import-button").should("not.exist");
+  afterEach(async () => {
+    const client = new AdminClient();
+    await client.deleteRealm(TEST_REALM);
   });
 
-  it("Import button reacts to loaded json", () => {
-    partialImportModal.open();
+  it("Opens and closes partial import dialog", () => {
+    modal.open();
+    modal.importButton().should("be.disabled");
+    modal.cancelButton().click();
+    modal.importButton().should("not.exist");
+  });
+
+  it("Import button only enabled if JSON has something to import", () => {
+    modal.open();
     cy.get("#partial-import-file").type("{}");
-    cy.getId("import-button").should("be.enabled");
+    modal.importButton().should("be.disabled");
+  });
+
+  it("Displays user options after multi-realm import", () => {
+    modal.open();
+    modal.typeResourceFile("multi-realm.json");
+
+    // Import button should be disabled if no checkboxes selected
+    modal.importButton().should("be.disabled");
+    modal.usersCheckbox().click();
+    modal.importButton().should("be.enabled");
+    modal.groupsCheckbox().click();
+    modal.importButton().should("be.enabled");
+    modal.groupsCheckbox().click();
+    modal.usersCheckbox().click();
+    modal.importButton().should("be.disabled");
+
+    // verify resource counts
+    modal.userCount().contains("1 users");
+    modal.groupCount().contains("1 groups");
+    modal.clientCount().contains("1 clients");
+    modal.idpCount().contains("1 identity providers");
+    modal.realmRolesCount().contains("2 realm roles");
+    modal.clientRolesCount().contains("1 client roles");
+
+    // import button should disable when switching realms
+    modal.usersCheckbox().click();
+    modal.importButton().should("be.enabled");
+    modal.selectRealm("realm2");
+    modal.importButton().should("be.disabled");
+
+    modal.clientCount().contains("2 clients");
+  });
+
+  it("Displays user options after realmless import", () => {
+    modal.open();
+
+    modal.typeResourceFile("client-only.json");
+
+    modal.realmSelector().should("not.exist");
+
+    modal.clientCount().contains("1 clients");
+
+    modal.userCount().should("not.exist");
+    modal.groupCount().should("not.exist");
+    modal.idpCount().should("not.exist");
+    modal.realmRolesCount().should("not.exist");
+    modal.clientRolesCount().should("not.exist");
   });
 
   // Unfortunately, the PatternFly FileUpload component does not create an id for the clear button.  So we can't easily test that function right now.

cypress/support/pages/admin_console/configure/realm_settings/PartialImportModal.ts

@@ -5,4 +5,65 @@ export default class GroupModal {
     cy.getId(this.openPartialImport).click();
     return this;
   }
+
+  typeResourceFile = (filename: string) => {
+    cy.readFile(
+      "cypress/integration/partial-import-test-data/" + filename
+    ).then((myJSON) => {
+      const text = JSON.stringify(myJSON);
+
+      cy.get("#partial-import-file").type(text, {
+        parseSpecialCharSequences: false,
+      });
+    });
+  };
+
+  importButton() {
+    return cy.getId("import-button");
+  }
+
+  cancelButton() {
+    return cy.getId("cancel-button");
+  }
+
+  groupsCheckbox() {
+    return cy.getId("groups-checkbox");
+  }
+
+  usersCheckbox() {
+    return cy.getId("users-checkbox");
+  }
+
+  userCount() {
+    return cy.getId("users-count");
+  }
+
+  clientCount() {
+    return cy.getId("clients-count");
+  }
+
+  groupCount() {
+    return cy.getId("groups-count");
+  }
+
+  idpCount() {
+    return cy.getId("identityProviders-count");
+  }
+
+  realmRolesCount() {
+    return cy.getId("realmRoles-count");
+  }
+
+  clientRolesCount() {
+    return cy.getId("clientRoles-count");
+  }
+
+  realmSelector() {
+    return cy.get("#realm-selector");
+  }
+
+  selectRealm(realm: string) {
+    this.realmSelector().click();
+    cy.getId(realm + "-select-option").click();
+  }
 }

src/realm-settings/PartialImport.tsx

@@ -1,11 +1,21 @@
 import React, { useState, useEffect } from "react";
 import { useTranslation } from "react-i18next";
+import _ from "lodash";
 import {
   Button,
   ButtonVariant,
+  DataList,
+  DataListCell,
+  DataListItem,
+  DataListItemCells,
+  DataListItemRow,
+  DataListCheck,
   Divider,
   Modal,
   ModalVariant,
+  Select,
+  SelectOption,
+  SelectOptionObject,
   Stack,
   StackItem,
   Text,
@@ -19,18 +29,254 @@ export type PartialImportProps = {
   toggleDialog: () => void;
 };
 
+// An imported JSON file can either be an array of realm objects
+// or a single realm object.
+type ImportedMultiRealm = [ImportedRealm?] | ImportedRealm;
+
+// Realms in imported json can have a lot more properties,
+// but these are the ones we care about.
+type ImportedRealm = {
+  id?: string;
+  realm?: string;
+  users?: [];
+  clients?: [];
+  groups?: [];
+  identityProviders?: [];
+  roles?: {
+    realm?: [];
+    client?: { [index: string]: [] };
+  };
+};
+
+type NonRoleResource = "users" | "clients" | "groups" | "identityProviders";
+type RoleResource = "realmRoles" | "clientRoles";
+type Resource = NonRoleResource | RoleResource;
+
+type CollisionOption = "FAIL" | "SKIP" | "OVERWRITE";
+
+type ResourceChecked = { [k in Resource]: boolean };
+
 export const PartialImportDialog = (props: PartialImportProps) => {
   const tRealm = useTranslation("realm-settings").t;
   const { t } = useTranslation("partial-import");
-  const [importEnabled, setImportEnabled] = useState(false);
 
-  // when dialog opens or closes, reset importEnabled to false
+  const [isFileSelected, setIsFileSelected] = useState(false);
+  const [isMultiRealm, setIsMultiRealm] = useState(false);
+  const [importedFile, setImportedFile] = useState<ImportedMultiRealm>([]);
+  const [isRealmSelectOpen, setIsRealmSelectOpen] = useState(false);
+  const [isCollisionSelectOpen, setIsCollisionSelectOpen] = useState(false);
+  const [collisionOption, setCollisionOption] = useState<CollisionOption>(
+    "FAIL"
+  );
+  const [targetRealm, setTargetRealm] = useState<ImportedRealm>({});
+
+  const allResourcesUnChecked: Readonly<ResourceChecked> = {
+    users: false,
+    clients: false,
+    groups: false,
+    identityProviders: false,
+    realmRoles: false,
+    clientRoles: false,
+  };
+
+  const [resourcesToImport, setResourcesToImport] = useState<ResourceChecked>(
+    _.cloneDeep(allResourcesUnChecked)
+  );
+
+  const [isAnyResourceChecked, setIsAnyResourceChecked] = useState(false);
+
+  const resetResourcesToImport = () => {
+    setResourcesToImport(_.cloneDeep(allResourcesUnChecked));
+    setIsAnyResourceChecked(false);
+  };
+
+  const resetInputState = () => {
+    setIsMultiRealm(false);
+    setImportedFile([]);
+    setTargetRealm({});
+    setCollisionOption("FAIL");
+    resetResourcesToImport();
+  };
+
+  // when dialog opens or closes, clear state
   useEffect(() => {
-    setImportEnabled(false);
+    setIsFileSelected(false);
+    resetInputState();
   }, [props.open]);
 
   const handleFileChange = (value: object) => {
-    setImportEnabled(!!value);
+    setIsFileSelected(!!value);
+    resetInputState();
+
+    setImportedFile(value);
+
+    if (value instanceof Array && value.length > 0) {
+      setIsMultiRealm(value.length > 1);
+      setTargetRealm(value[0] || {});
+    } else {
+      setIsMultiRealm(false);
+      setTargetRealm((value as ImportedRealm) || {});
+    }
+  };
+
+  const handleRealmSelect = (
+    event: React.ChangeEvent<Element> | React.MouseEvent<Element, MouseEvent>,
+    realm: string | SelectOptionObject
+  ) => {
+    setTargetRealm(realm as ImportedRealm);
+    setIsRealmSelectOpen(false);
+    resetResourcesToImport();
+  };
+
+  const handleResourceCheckBox = (
+    checked: boolean,
+    event: React.FormEvent<HTMLInputElement>
+  ) => {
+    const resource: Resource = event.currentTarget.name as Resource;
+    const copyOfResourcesToImport = _.cloneDeep(resourcesToImport);
+    copyOfResourcesToImport[resource] = checked;
+    setResourcesToImport(copyOfResourcesToImport);
+    setIsAnyResourceChecked(resourcesChecked(copyOfResourcesToImport));
+  };
+
+  const realmSelectOptions = () => {
+    if (!isMultiRealm) return [];
+
+    const mapper = (realm: ImportedRealm) => {
+      return (
+        <SelectOption
+          key={realm.id}
+          value={realm}
+          data-testid={realm.id + "-select-option"}
+        >
+          {realm.realm || realm.id}
+        </SelectOption>
+      );
+    };
+
+    return (importedFile as [ImportedRealm]).map(mapper);
+  };
+
+  const handleCollisionSelect = (
+    event: React.ChangeEvent<Element> | React.MouseEvent<Element, MouseEvent>,
+    option: string | SelectOptionObject
+  ) => {
+    setCollisionOption(option as CollisionOption);
+    setIsCollisionSelectOpen(false);
+  };
+
+  const collisionOptions = () => {
+    return [
+      <SelectOption key="fail" value="FAIL">
+        {t("FAIL")}
+      </SelectOption>,
+      <SelectOption key="skip" value="SKIP">
+        {t("SKIP")}
+      </SelectOption>,
+      <SelectOption key="overwrite" value="OVERWRITE">
+        {t("OVERWRITE")}
+      </SelectOption>,
+    ];
+  };
+
+  const targetHasResources = () => {
+    return (
+      targetHasResource("users") ||
+      targetHasResource("groups") ||
+      targetHasResource("clients") ||
+      targetHasResource("identityProviders") ||
+      targetHasRealmRoles() ||
+      targetHasClientRoles()
+    );
+  };
+
+  const targetHasResource = (resource: NonRoleResource) => {
+    return (
+      targetRealm &&
+      targetRealm[resource] instanceof Array &&
+      targetRealm[resource]!.length > 0
+    );
+  };
+
+  const targetHasRoles = () => {
+    return (
+      targetRealm && Object.prototype.hasOwnProperty.call(targetRealm, "roles")
+    );
+  };
+
+  const targetHasRealmRoles = () => {
+    return (
+      targetHasRoles() &&
+      targetRealm.roles!.realm instanceof Array &&
+      targetRealm.roles!.realm.length > 0
+    );
+  };
+
+  const targetHasClientRoles = () => {
+    return (
+      targetHasRoles() &&
+      Object.prototype.hasOwnProperty.call(targetRealm.roles, "client") &&
+      Object.keys(targetRealm.roles!.client!).length > 0
+    );
+  };
+
+  const itemCount = (resource: Resource) => {
+    if (!isFileSelected) return 0;
+
+    if (targetHasRealmRoles() && resource === "realmRoles")
+      return targetRealm.roles!.realm!.length;
+
+    if (targetHasClientRoles() && resource == "clientRoles")
+      return clientRolesCount(targetRealm.roles!.client!);
+
+    if (!targetRealm[resource as NonRoleResource]) return 0;
+
+    return targetRealm[resource as NonRoleResource]!.length;
+  };
+
+  const clientRolesCount = (clientRoles: { [index: string]: [] }) => {
+    let total = 0;
+    for (const clientName in clientRoles) {
+      total += clientRoles[clientName].length;
+    }
+    return total;
+  };
+
+  const resourcesChecked = (resources: ResourceChecked) => {
+    let resource: Resource;
+    for (resource in resources) {
+      if (resources[resource]) return true;
+    }
+
+    return false;
+  };
+
+  const resourceDataListItem = (
+    resource: Resource,
+    resourceDisplayName: string
+  ) => {
+    return (
+      <DataListItem aria-labelledby={`${resource}-list-item`}>
+        <DataListItemRow>
+          <DataListCheck
+            aria-labelledby={`${resource}-checkbox`}
+            name={resource}
+            isChecked={resourcesToImport[resource]}
+            onChange={handleResourceCheckBox}
+            data-testid={resource + "-checkbox"}
+          />
+          <DataListItemCells
+            dataListCells={[
+              <DataListCell key={resource}>
+                <span data-testid={resource + "-count"}>
+                  {itemCount(resource)} {resourceDisplayName}
+                </span>
+              </DataListCell>,
+            ]}
+          />
+        </DataListItemRow>
+      </DataListItem>
+    );
   };
 
   return (
@@ -44,7 +290,7 @@ export const PartialImportDialog = (props: PartialImportProps) => {
           id="modal-import"
           data-testid="import-button"
           key="import"
-          isDisabled={!importEnabled}
+          isDisabled={!isAnyResourceChecked}
           onClick={() => {
             props.toggleDialog();
           }}
@@ -76,14 +322,61 @@ export const PartialImportDialog = (props: PartialImportProps) => {
             onChange={handleFileChange}
           />
         </StackItem>
-        {importEnabled && (
-          <StackItem>
-            <Divider />
-            TODO: This section will include{" "}
-            <strong>Choose the resources...</strong> and{" "}
-            <strong>If a resource already exists....</strong>
-            <Divider />
-          </StackItem>
+
+        {isFileSelected && targetHasResources() && (
+          <>
+            <StackItem>
+              <Divider />
+            </StackItem>
+            {isMultiRealm && (
+              <StackItem>
+                <Text>{t("selectRealm")}:</Text>
+                <Select
+                  toggleId="realm-selector"
+                  isOpen={isRealmSelectOpen}
+                  onToggle={() => setIsRealmSelectOpen(!isRealmSelectOpen)}
+                  onSelect={handleRealmSelect}
+                  placeholderText={targetRealm.realm || targetRealm.id}
+                >
+                  {realmSelectOptions()}
+                </Select>
+              </StackItem>
+            )}
+            <StackItem>
+              <Text>{t("chooseResources")}:</Text>
+              <DataList aria-label="Resources to import" isCompact>
+                {targetHasResource("users") &&
+                  resourceDataListItem("users", "users")}
+                {targetHasResource("groups") &&
+                  resourceDataListItem("groups", "groups")}
+                {targetHasResource("clients") &&
+                  resourceDataListItem("clients", "clients")}
+                {targetHasResource("identityProviders") &&
+                  resourceDataListItem(
+                    "identityProviders",
+                    "identity providers"
+                  )}
+                {targetHasRealmRoles() &&
+                  resourceDataListItem("realmRoles", "realm roles")}
+                {targetHasClientRoles() &&
+                  resourceDataListItem("clientRoles", "client roles")}
+              </DataList>
+            </StackItem>
+            <StackItem>
+              <Text>{t("selectIfResourceExists")}:</Text>
+              <Select
+                isOpen={isCollisionSelectOpen}
+                direction="up"
+                onToggle={() => {
+                  setIsCollisionSelectOpen(!isCollisionSelectOpen);
+                }}
+                onSelect={handleCollisionSelect}
+                placeholderText={t(collisionOption)}
+              >
+                {collisionOptions()}
+              </Select>
+            </StackItem>
+          </>
         )}
       </Stack>
     </Modal>

src/realm-settings/messages.json

@@ -1,4 +1,3 @@
-  
 {
   "realm-settings": {
     "partialImport": "Partial import",
@@ -477,7 +476,6 @@
         "name": "Refresh token error",
         "description": "Refresh token error"
       }
-
     },
     "emptyEvents": "Nothing to add",
     "emptyEventsInstructions": "There are no more events types left to add",
@@ -494,8 +492,14 @@
     "confirm": "Confirm"
   },
   "partial-import": {
-    "partialImportHeaderText": "Partial import allows you to import users, clients, and resources from a previously exported json file.",
-    "import": "Import"
+    "partialImportHeaderText": "Partial import allows you to import users, clients, and other resources from a previously exported json file.",
+    "selectRealm": "Select realm",
+    "chooseResources": "Choose the resources you want to import",
+    "selectIfResourceExists": "If a resource already exists, specify what should be done",
+    "import": "Import",
+    "FAIL": "Fail import",
+    "SKIP": "Skip",
+    "OVERWRITE": "Overwrite"
   },
   "onDragStart": "Dragging started for item {{id}}",
   "onDragMove": "Dragging item {{id}}",