Add migration tests for AuroraDB (#31396)

Fixes #31024
Signed-off-by: Kamesh Akella <kamesh.asp@gmail.com>

.github/scripts/ansible/aws_ec2.sh

@@ -8,6 +8,7 @@ fi
 
 OPERATION=$1
 REGION=$2
+CLUSTER_NAME=$3
 
 case $OPERATION in
   requirements)
@@ -16,8 +17,7 @@ case $OPERATION in
   ;;
   create|delete|start|stop)
     if [ -f "env.yml" ]; then ANSIBLE_CUSTOM_VARS_ARG="-e @env.yml"; fi
-    CLUSTER_NAME=${CLUSTER_NAME:-"keycloak_$(whoami)"}
-    ansible-playbook aws_ec2.yml -v -e "region=$REGION" -e "operation=$OPERATION" -e "cluster_name=$CLUSTER_NAME" $ANSIBLE_CUSTOM_VARS_ARG "${@:3}"
+    ansible-playbook aws_ec2.yml -v -e "region=$REGION" -e "operation=$OPERATION" -e "cluster_name=$CLUSTER_NAME" $ANSIBLE_CUSTOM_VARS_ARG "${@:4}"
   ;;
   *)
     echo "Invalid option!"

.github/scripts/ansible/keycloak_ec2_installer.sh

@@ -7,9 +7,8 @@ if [[ "$RUNNER_DEBUG" == "1" ]]; then
 fi
 
 REGION=$1
-KEYCLOAK_SRC=$2
-
-CLUSTER_NAME=${CLUSTER_NAME:-"keycloak_$(whoami)"}
+CLUSTER_NAME=$2
+KEYCLOAK_SRC=$3
 
 ansible-playbook -i ${CLUSTER_NAME}_${REGION}_inventory.yml keycloak.yml \
   -e "keycloak_src=\"${KEYCLOAK_SRC}\""

.github/scripts/ansible/mvn_ec2_runner.sh

@@ -7,9 +7,10 @@ if [[ "$RUNNER_DEBUG" == "1" ]]; then
 fi
 
 REGION=$1
-MVN_PARAMS=${@:2}
+CLUSTER_NAME=$2
+MVN_PARAMS=${@:3}
 
-CLUSTER_NAME=${CLUSTER_NAME:-"keycloak_$(whoami)"}
+echo "mvn_params=\"${MVN_PARAMS}\""
 
 ansible-playbook -i ${CLUSTER_NAME}_${REGION}_inventory.yml mvn.yml \
   -e "mvn_params=\"${MVN_PARAMS}\""

.github/scripts/ansible/roles/aws_ec2/tasks/create-resources.yml

@@ -19,6 +19,13 @@
   register: group
   no_log: "{{ no_log_sensitive }}"
 
+- name: Delete existing key pair if it exists
+  amazon.aws.ec2_key:
+    region: '{{ region }}'
+    name: '{{ cluster_name }}'
+    state: absent
+  ignore_errors: true
+
 - name: Create Key
   amazon.aws.ec2_key:
     state: present

.github/scripts/ansible/roles/mvn_ec2_runner/tasks/run.yml

@@ -16,6 +16,7 @@
   ansible.builtin.shell: |
     set -o pipefail
     cd {{ kc_home }}
+    echo "{{ mvn_params }}"
     ./mvnw {{ mvn_params }}
   args:
     executable: /usr/bin/bash

.github/workflows/ci.yml

@@ -33,8 +33,9 @@ jobs:
       ci-store: ${{ steps.conditional.outputs.ci-store }}
       ci-sssd: ${{ steps.conditional.outputs.ci-sssd }}
       ci-webauthn: ${{ steps.conditional.outputs.ci-webauthn }}
-      ci-store-matrix: ${{ steps.conditional-stores.outputs.matrix }}
       ci-test-poc: ${{ steps.conditional.outputs.ci-test-poc }}
+      ci-aurora: ${{ steps.auroradb-tests.outputs.run-aurora-tests }}
+
     steps:
       - uses: actions/checkout@v4
 
@@ -43,13 +44,14 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
 
-      - id: conditional-stores
+      - name: AuroraDB conditional check
+        id: auroradb-tests
         run: |
-          STORES="postgres, mysql, oracle, mssql, mariadb"
+          RUN_AURORADB_TESTS=false
           if [[ $GITHUB_EVENT_NAME != "pull_request" && -n "${{ secrets.AWS_SECRET_ACCESS_KEY }}" ]]; then
-            STORES+=", aurora-postgres"
+            RUN_AURORADB_TESTS=true
           fi
-          echo "matrix=$(echo $STORES  | jq -Rc 'split(", ")')" >> $GITHUB_OUTPUT
+          echo "run-aurora-tests=$RUN_AURORADB_TESTS" >> $GITHUB_OUTPUT
 
   build:
     name: Build
@@ -420,85 +422,184 @@ jobs:
         with:
           job-id: remote-infinispan-integration-tests
 
-  store-integration-tests:
-    name: Store IT
-    needs: [build, conditional]
-    if: needs.conditional.outputs.ci-store == 'true'
+  auroradb-integration-tests:
+    name: AuroraDB IT
+    needs: conditional
+    if: needs.conditional.outputs.ci-aurora == 'true'
     runs-on: ubuntu-latest
     timeout-minutes: 150
-    strategy:
-      matrix:
-        db: ${{ fromJson(needs.conditional.outputs.ci-store-matrix) }}
-      fail-fast: false
     steps:
       - uses: actions/checkout@v4
 
       - id: aurora-init
         name: Initialize Aurora environment
-        if: ${{ matrix.db == 'aurora-postgres' }}
         run: |
           AWS_REGION=us-east-1
-          echo "Region: ${AWS_REGION}"
+          echo "AWS Region: ${AWS_REGION}"
           
           aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws configure set region ${AWS_REGION}
+          
+          AURORA_CLUSTER_NAME="gh-action-$(git rev-parse --short HEAD)-${{ github.run_id }}-${{ github.run_attempt }}"
           PASS=$(tr -dc A-Za-z0-9 </dev/urandom | head -c 13; echo)
           echo "::add-mask::${PASS}"
           
-          echo "name=gh-action-$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-          echo "password=${PASS}" >> $GITHUB_OUTPUT
+          echo "aurora-cluster-name=${AURORA_CLUSTER_NAME}" >> $GITHUB_OUTPUT
+          echo "aurora-cluster-password=${PASS}" >> $GITHUB_OUTPUT
           echo "region=${AWS_REGION}" >> $GITHUB_OUTPUT
+          curl --fail-with-body https://truststore.pki.rds.amazonaws.com/${AWS_REGION}/${AWS_REGION}-bundle.pem -o aws.pem
+          PROPS+=' -Dkeycloak.connectionsJpa.jdbcParameters=\"?ssl=true&sslmode=verify-ca&sslrootcert=/opt/keycloak/aws.pem\"'
+          
+          echo "maven_properties=${PROPS}" >> $GITHUB_OUTPUT
 
       - id: aurora-create
         name: Create Aurora DB
-        if: ${{ matrix.db == 'aurora-postgres' }}
         uses: ./.github/actions/aurora-create-database
         with:
-          name: ${{ steps.aurora-init.outputs.name }}
-          password: ${{ steps.aurora-init.outputs.password }}
+          name: ${{ steps.aurora-init.outputs.aurora-cluster-name }}
+          password: ${{ steps.aurora-init.outputs.aurora-cluster-password }}
           region: ${{ steps.aurora-init.outputs.region }}
 
-      - id: integration-test-setup
-        name: Integration test setup
-        if: ${{ matrix.db != 'aurora-postgres' }}
-        uses: ./.github/actions/integration-test-setup
-
-      - name: Run Aurora tests on EC2
-        id: aurora-tests
-        if: ${{ matrix.db == 'aurora-postgres' }}
+      - id: ec2-create
+        name: Create EC2 runner instance
         run: |
-          PROPS="-Dauth.server.db.host=${{ steps.aurora-create.outputs.endpoint }}"
-          PROPS+=" -Dkeycloak.connectionsJpa.password=${{ steps.aurora-init.outputs.password }}"
-
-          REGION=${{ steps.aurora-init.outputs.region }}
-
-          curl --fail-with-body https://truststore.pki.rds.amazonaws.com/${REGION}/${REGION}-bundle.pem -o aws.pem
-          PROPS+=" -Dkeycloak.connectionsJpa.jdbcParameters=\"?ssl=true&sslmode=verify-ca&sslrootcert=/opt/keycloak/aws.pem\""
-
-          TESTS=`testsuite/integration-arquillian/tests/base/testsuites/suite.sh database`
-          echo "Tests: $TESTS"
+          AWS_REGION=${{ steps.aurora-init.outputs.region }}
+          EC2_CLUSTER_NAME=keycloak_$(git rev-parse --short HEAD)
+          echo "ec2_cluster=${EC2_CLUSTER_NAME}" >> $GITHUB_OUTPUT
           
           git archive --format=zip --output /tmp/keycloak.zip $GITHUB_REF
           zip -u /tmp/keycloak.zip aws.pem
  
           cd .github/scripts/ansible
-          export CLUSTER_NAME=keycloak_$(git rev-parse --short HEAD)
-          echo "ec2_cluster=${CLUSTER_NAME}" >> $GITHUB_OUTPUT
           ./aws_ec2.sh requirements
-          ./aws_ec2.sh create ${REGION}
-          ./keycloak_ec2_installer.sh ${REGION} /tmp/keycloak.zip
-          ./mvn_ec2_runner.sh ${REGION} "clean install -B -DskipTests -Pdistribution"
-          ./mvn_ec2_runner.sh ${REGION} "clean install -B -DskipTests -pl testsuite/integration-arquillian/servers/auth-server/quarkus -Pauth-server-quarkus -Pdb-aurora-postgres -Dmaven.build.cache.enabled=true"
-          ./mvn_ec2_runner.sh ${REGION} "test -B ${{ env.SUREFIRE_RETRY }} -Pauth-server-quarkus -Pdb-${{ matrix.db }} $PROPS -Dtest=$TESTS -pl testsuite/integration-arquillian/tests/base 2>&1 | misc/log/trimmer.sh"
+          ./aws_ec2.sh create ${AWS_REGION} ${EC2_CLUSTER_NAME}
+          ./keycloak_ec2_installer.sh ${AWS_REGION} ${EC2_CLUSTER_NAME} /tmp/keycloak.zip
+          ./mvn_ec2_runner.sh ${AWS_REGION} ${EC2_CLUSTER_NAME} "clean install -B -DskipTests -Pdistribution"
+          ./mvn_ec2_runner.sh ${AWS_REGION} ${EC2_CLUSTER_NAME} "clean install -B -DskipTests -pl testsuite/integration-arquillian/servers/auth-server/quarkus -Pauth-server-quarkus -Pdb-aurora-postgres -Dmaven.build.cache.enabled=true"
+
+      - name: Run Aurora migration tests on EC2
+        id: aurora-migration-tests
+        env:
+          old-version: 24.0.4
+        run: |
+          EC2_CLUSTER_NAME=${{ steps.ec2-create.outputs.ec2_cluster }}
+          AWS_REGION=${{ steps.aurora-init.outputs.region }}
+          PROPS='${{ steps.aurora-init.outputs.maven_properties }}'
+          
+          PROPS+=" -Dauth.server.db.host=${{ steps.aurora-create.outputs.endpoint }} -Dkeycloak.connectionsJpa.password=${{ steps.aurora-init.outputs.aurora-cluster-password }}"
+          PROPS+=" -Djdbc.mvn.groupId=software.amazon.jdbc -Djdbc.mvn.artifactId=aws-advanced-jdbc-wrapper -Djdbc.mvn.version=2.3.1 -Djdbc.driver.tmp.dir=target/unpacked/keycloak-${{ env.old-version }}/providers"
+
+          cd .github/scripts/ansible
+          ./mvn_ec2_runner.sh ${AWS_REGION} ${EC2_CLUSTER_NAME} "clean install -B ${{ env.SUREFIRE_RETRY }} -Pauth-server-quarkus -Pdb-aurora-postgres -Pauth-server-migration $PROPS -Dtest=MigrationTest -Dmigration.mode=auto -Dmigrated.auth.server.version=${{ env.old-version }} -Dmigration.import.file.name=migration-realm-${{ env.old-version }}.json -Dauth.server.ssl.required=false -f testsuite/integration-arquillian/pom.xml 2>&1 | misc/log/trimmer.sh"
+
+          # Copy returned surefire-report directories to workspace root to ensure they're discovered
+          results=(files/keycloak/results/*)
+          rsync -a $results/* ../../../
+
+          rm -rf $results
+
+      - name: Upload JVM Heapdumps
+        if: always()
+        uses: ./.github/actions/upload-heapdumps
+
+      - uses: ./.github/actions/upload-flaky-tests
+        name: Upload flaky tests
+        env:
+          GH_TOKEN: ${{ github.token }}
+        with:
+          job-name: AuroraDB IT
+
+      - name: Surefire reports
+        if: always()
+        uses: ./.github/actions/archive-surefire-reports
+        with:
+          job-id: migration-tests-${{ env.old-version }}-aurora-postgres
+
+      - name: EC2 Maven Logs
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: auroraDB-migration-tests-mvn-logs
+          path: .github/scripts/ansible/files
+
+      - name: Run Aurora integration tests on EC2
+        id: aurora-integration-tests
+        run: |
+          EC2_CLUSTER_NAME=${{ steps.ec2-create.outputs.ec2_cluster }}
+          AWS_REGION=${{ steps.aurora-init.outputs.region }}
+          PROPS='${{ steps.aurora-init.outputs.maven_properties }}'          
+          PROPS+=" -Dauth.server.db.host=${{ steps.aurora-create.outputs.endpoint }} -Dkeycloak.connectionsJpa.password=${{ steps.aurora-init.outputs.aurora-cluster-password }}"
+          
+          TESTS=`testsuite/integration-arquillian/tests/base/testsuites/suite.sh database`
+          echo "Tests: $TESTS"
+          
+          cd .github/scripts/ansible
+          ./mvn_ec2_runner.sh ${AWS_REGION} ${EC2_CLUSTER_NAME} "test -B ${{ env.SUREFIRE_RETRY }} -Pauth-server-quarkus -Pdb-aurora-postgres $PROPS -Dtest=$TESTS -pl testsuite/integration-arquillian/tests/base 2>&1 | misc/log/trimmer.sh"
           
           # Copy returned surefire-report directories to workspace root to ensure they're discovered
           results=(files/keycloak/results/*)
           rsync -a $results/* ../../../
           rm -rf $results
 
+      - name: Upload JVM Heapdumps
+        if: always()
+        uses: ./.github/actions/upload-heapdumps
+
+      - uses: ./.github/actions/upload-flaky-tests
+        name: Upload flaky tests
+        env:
+          GH_TOKEN: ${{ github.token }}
+        with:
+          job-name: AuroraDB IT
+
+      - name: Surefire reports
+        if: always()
+        uses: ./.github/actions/archive-surefire-reports
+        with:
+          job-id: aurora-integration-tests
+
+      - name: EC2 Maven Logs
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: aurora-integration-tests-mvn-logs
+          path: .github/scripts/ansible/files
+
+      - name: Delete EC2 Instance
+        if: always()
+        working-directory: .github/scripts/ansible
+        run: |
+          ./aws_ec2.sh delete ${{ steps.aurora-init.outputs.region }} ${{ steps.ec2-create.outputs.ec2_cluster }}
+
+      - name: Delete Aurora DB
+        if: always()
+        run: |
+          gh workflow run aurora-delete.yml \
+            -f name=${{ steps.aurora-init.outputs.aurora-cluster-name }} \
+            -f region=${{ steps.aurora-init.outputs.region }} \
+            --repo ${{ github.repository }} \
+            --ref ${{ github.ref_name }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+  store-integration-tests:
+    name: Store IT
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 75
+    strategy:
+      matrix:
+        db: [postgres, mysql, oracle, mssql, mariadb]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v4
+
+      - id: integration-test-setup
+        name: Integration test setup
+        uses: ./.github/actions/integration-test-setup
+
       - name: Run base tests
-        if: ${{ matrix.db != 'aurora-postgres' }}
         run: |
           TESTS=`testsuite/integration-arquillian/tests/base/testsuites/suite.sh database`
           echo "Tests: $TESTS"
@@ -521,31 +622,6 @@ jobs:
         with:
           job-id: store-integration-tests-${{ matrix.db }}
 
-      - name: EC2 Maven Logs
-        if: failure()
-        uses: actions/upload-artifact@v4
-        with:
-          name: store-it-mvn-logs
-          path: .github/scripts/ansible/files
-
-      - name: Delete Aurora EC2 Instance
-        if: ${{ always() && matrix.db == 'aurora-postgres' }}
-        working-directory: .github/scripts/ansible
-        run: |
-          export CLUSTER_NAME=${{ steps.aurora-tests.outputs.ec2_cluster }}
-          ./aws_ec2.sh delete ${{ steps.aurora-init.outputs.region }}
-
-      - name: Delete Aurora DB
-        if: ${{ always() && matrix.db == 'aurora-postgres' }}
-        run: |
-          gh workflow run aurora-delete.yml \
-            -f name=${{ steps.aurora-init.outputs.name }} \
-            -f region=${{ steps.aurora-init.outputs.region }} \
-            --repo ${{ github.repository }} \
-            --ref ${{ github.ref_name }}
-        env:
-          GH_TOKEN: ${{ github.token }}
-
   store-model-tests:
     name: Store Model Tests
     runs-on: ubuntu-latest

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/containers/KeycloakQuarkusServerDeployableContainer.java

@@ -83,8 +83,8 @@ public class KeycloakQuarkusServerDeployableContainer extends AbstractQuarkusDep
         ProcessBuilder pb = new ProcessBuilder(commands);
         Process p = pb.directory(wrkDir).inheritIO().start();
         try {
-            if (!p.waitFor(60, TimeUnit.SECONDS)) {
-                throw new IOException("Command " + command + " did not finished in 60 seconds");
+            if (!p.waitFor(300, TimeUnit.SECONDS)) {
+                throw new IOException("Command " + command + " did not finished in 300 seconds");
             }
             if (p.exitValue() != 0) {
                 throw new IOException("Command " + command + " was executed with exit status " + p.exitValue());