Add migration tests for AuroraDB (#31396)
Fixes #31024 Signed-off-by: Kamesh Akella <kamesh.asp@gmail.com>
This commit is contained in:
parent
30be268672
commit
33b3fd313c
7 changed files with 168 additions and 84 deletions
4
.github/scripts/ansible/aws_ec2.sh
vendored
4
.github/scripts/ansible/aws_ec2.sh
vendored
|
@ -8,6 +8,7 @@ fi
|
|||
|
||||
OPERATION=$1
|
||||
REGION=$2
|
||||
CLUSTER_NAME=$3
|
||||
|
||||
case $OPERATION in
|
||||
requirements)
|
||||
|
@ -16,8 +17,7 @@ case $OPERATION in
|
|||
;;
|
||||
create|delete|start|stop)
|
||||
if [ -f "env.yml" ]; then ANSIBLE_CUSTOM_VARS_ARG="-e @env.yml"; fi
|
||||
CLUSTER_NAME=${CLUSTER_NAME:-"keycloak_$(whoami)"}
|
||||
ansible-playbook aws_ec2.yml -v -e "region=$REGION" -e "operation=$OPERATION" -e "cluster_name=$CLUSTER_NAME" $ANSIBLE_CUSTOM_VARS_ARG "${@:3}"
|
||||
ansible-playbook aws_ec2.yml -v -e "region=$REGION" -e "operation=$OPERATION" -e "cluster_name=$CLUSTER_NAME" $ANSIBLE_CUSTOM_VARS_ARG "${@:4}"
|
||||
;;
|
||||
*)
|
||||
echo "Invalid option!"
|
||||
|
|
|
@ -7,9 +7,8 @@ if [[ "$RUNNER_DEBUG" == "1" ]]; then
|
|||
fi
|
||||
|
||||
REGION=$1
|
||||
KEYCLOAK_SRC=$2
|
||||
|
||||
CLUSTER_NAME=${CLUSTER_NAME:-"keycloak_$(whoami)"}
|
||||
CLUSTER_NAME=$2
|
||||
KEYCLOAK_SRC=$3
|
||||
|
||||
ansible-playbook -i ${CLUSTER_NAME}_${REGION}_inventory.yml keycloak.yml \
|
||||
-e "keycloak_src=\"${KEYCLOAK_SRC}\""
|
||||
|
|
5
.github/scripts/ansible/mvn_ec2_runner.sh
vendored
5
.github/scripts/ansible/mvn_ec2_runner.sh
vendored
|
@ -7,9 +7,10 @@ if [[ "$RUNNER_DEBUG" == "1" ]]; then
|
|||
fi
|
||||
|
||||
REGION=$1
|
||||
MVN_PARAMS=${@:2}
|
||||
CLUSTER_NAME=$2
|
||||
MVN_PARAMS=${@:3}
|
||||
|
||||
CLUSTER_NAME=${CLUSTER_NAME:-"keycloak_$(whoami)"}
|
||||
echo "mvn_params=\"${MVN_PARAMS}\""
|
||||
|
||||
ansible-playbook -i ${CLUSTER_NAME}_${REGION}_inventory.yml mvn.yml \
|
||||
-e "mvn_params=\"${MVN_PARAMS}\""
|
||||
|
|
|
@ -19,6 +19,13 @@
|
|||
register: group
|
||||
no_log: "{{ no_log_sensitive }}"
|
||||
|
||||
- name: Delete existing key pair if it exists
|
||||
amazon.aws.ec2_key:
|
||||
region: '{{ region }}'
|
||||
name: '{{ cluster_name }}'
|
||||
state: absent
|
||||
ignore_errors: true
|
||||
|
||||
- name: Create Key
|
||||
amazon.aws.ec2_key:
|
||||
state: present
|
||||
|
|
|
@ -16,6 +16,7 @@
|
|||
ansible.builtin.shell: |
|
||||
set -o pipefail
|
||||
cd {{ kc_home }}
|
||||
echo "{{ mvn_params }}"
|
||||
./mvnw {{ mvn_params }}
|
||||
args:
|
||||
executable: /usr/bin/bash
|
||||
|
|
226
.github/workflows/ci.yml
vendored
226
.github/workflows/ci.yml
vendored
|
@ -33,8 +33,9 @@ jobs:
|
|||
ci-store: ${{ steps.conditional.outputs.ci-store }}
|
||||
ci-sssd: ${{ steps.conditional.outputs.ci-sssd }}
|
||||
ci-webauthn: ${{ steps.conditional.outputs.ci-webauthn }}
|
||||
ci-store-matrix: ${{ steps.conditional-stores.outputs.matrix }}
|
||||
ci-test-poc: ${{ steps.conditional.outputs.ci-test-poc }}
|
||||
ci-aurora: ${{ steps.auroradb-tests.outputs.run-aurora-tests }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
|
@ -43,13 +44,14 @@ jobs:
|
|||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- id: conditional-stores
|
||||
- name: AuroraDB conditional check
|
||||
id: auroradb-tests
|
||||
run: |
|
||||
STORES="postgres, mysql, oracle, mssql, mariadb"
|
||||
RUN_AURORADB_TESTS=false
|
||||
if [[ $GITHUB_EVENT_NAME != "pull_request" && -n "${{ secrets.AWS_SECRET_ACCESS_KEY }}" ]]; then
|
||||
STORES+=", aurora-postgres"
|
||||
RUN_AURORADB_TESTS=true
|
||||
fi
|
||||
echo "matrix=$(echo $STORES | jq -Rc 'split(", ")')" >> $GITHUB_OUTPUT
|
||||
echo "run-aurora-tests=$RUN_AURORADB_TESTS" >> $GITHUB_OUTPUT
|
||||
|
||||
build:
|
||||
name: Build
|
||||
|
@ -420,85 +422,184 @@ jobs:
|
|||
with:
|
||||
job-id: remote-infinispan-integration-tests
|
||||
|
||||
store-integration-tests:
|
||||
name: Store IT
|
||||
needs: [build, conditional]
|
||||
if: needs.conditional.outputs.ci-store == 'true'
|
||||
auroradb-integration-tests:
|
||||
name: AuroraDB IT
|
||||
needs: conditional
|
||||
if: needs.conditional.outputs.ci-aurora == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 150
|
||||
strategy:
|
||||
matrix:
|
||||
db: ${{ fromJson(needs.conditional.outputs.ci-store-matrix) }}
|
||||
fail-fast: false
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- id: aurora-init
|
||||
name: Initialize Aurora environment
|
||||
if: ${{ matrix.db == 'aurora-postgres' }}
|
||||
run: |
|
||||
AWS_REGION=us-east-1
|
||||
echo "Region: ${AWS_REGION}"
|
||||
|
||||
echo "AWS Region: ${AWS_REGION}"
|
||||
|
||||
aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
aws configure set region ${AWS_REGION}
|
||||
|
||||
AURORA_CLUSTER_NAME="gh-action-$(git rev-parse --short HEAD)-${{ github.run_id }}-${{ github.run_attempt }}"
|
||||
PASS=$(tr -dc A-Za-z0-9 </dev/urandom | head -c 13; echo)
|
||||
echo "::add-mask::${PASS}"
|
||||
|
||||
echo "name=gh-action-$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
|
||||
echo "password=${PASS}" >> $GITHUB_OUTPUT
|
||||
|
||||
echo "aurora-cluster-name=${AURORA_CLUSTER_NAME}" >> $GITHUB_OUTPUT
|
||||
echo "aurora-cluster-password=${PASS}" >> $GITHUB_OUTPUT
|
||||
echo "region=${AWS_REGION}" >> $GITHUB_OUTPUT
|
||||
curl --fail-with-body https://truststore.pki.rds.amazonaws.com/${AWS_REGION}/${AWS_REGION}-bundle.pem -o aws.pem
|
||||
PROPS+=' -Dkeycloak.connectionsJpa.jdbcParameters=\"?ssl=true&sslmode=verify-ca&sslrootcert=/opt/keycloak/aws.pem\"'
|
||||
|
||||
echo "maven_properties=${PROPS}" >> $GITHUB_OUTPUT
|
||||
|
||||
- id: aurora-create
|
||||
name: Create Aurora DB
|
||||
if: ${{ matrix.db == 'aurora-postgres' }}
|
||||
uses: ./.github/actions/aurora-create-database
|
||||
with:
|
||||
name: ${{ steps.aurora-init.outputs.name }}
|
||||
password: ${{ steps.aurora-init.outputs.password }}
|
||||
name: ${{ steps.aurora-init.outputs.aurora-cluster-name }}
|
||||
password: ${{ steps.aurora-init.outputs.aurora-cluster-password }}
|
||||
region: ${{ steps.aurora-init.outputs.region }}
|
||||
|
||||
- id: integration-test-setup
|
||||
name: Integration test setup
|
||||
if: ${{ matrix.db != 'aurora-postgres' }}
|
||||
uses: ./.github/actions/integration-test-setup
|
||||
|
||||
- name: Run Aurora tests on EC2
|
||||
id: aurora-tests
|
||||
if: ${{ matrix.db == 'aurora-postgres' }}
|
||||
- id: ec2-create
|
||||
name: Create EC2 runner instance
|
||||
run: |
|
||||
PROPS="-Dauth.server.db.host=${{ steps.aurora-create.outputs.endpoint }}"
|
||||
PROPS+=" -Dkeycloak.connectionsJpa.password=${{ steps.aurora-init.outputs.password }}"
|
||||
|
||||
REGION=${{ steps.aurora-init.outputs.region }}
|
||||
|
||||
curl --fail-with-body https://truststore.pki.rds.amazonaws.com/${REGION}/${REGION}-bundle.pem -o aws.pem
|
||||
PROPS+=" -Dkeycloak.connectionsJpa.jdbcParameters=\"?ssl=true&sslmode=verify-ca&sslrootcert=/opt/keycloak/aws.pem\""
|
||||
|
||||
TESTS=`testsuite/integration-arquillian/tests/base/testsuites/suite.sh database`
|
||||
echo "Tests: $TESTS"
|
||||
|
||||
AWS_REGION=${{ steps.aurora-init.outputs.region }}
|
||||
EC2_CLUSTER_NAME=keycloak_$(git rev-parse --short HEAD)
|
||||
echo "ec2_cluster=${EC2_CLUSTER_NAME}" >> $GITHUB_OUTPUT
|
||||
|
||||
git archive --format=zip --output /tmp/keycloak.zip $GITHUB_REF
|
||||
zip -u /tmp/keycloak.zip aws.pem
|
||||
|
||||
cd .github/scripts/ansible
|
||||
./aws_ec2.sh requirements
|
||||
./aws_ec2.sh create ${AWS_REGION} ${EC2_CLUSTER_NAME}
|
||||
./keycloak_ec2_installer.sh ${AWS_REGION} ${EC2_CLUSTER_NAME} /tmp/keycloak.zip
|
||||
./mvn_ec2_runner.sh ${AWS_REGION} ${EC2_CLUSTER_NAME} "clean install -B -DskipTests -Pdistribution"
|
||||
./mvn_ec2_runner.sh ${AWS_REGION} ${EC2_CLUSTER_NAME} "clean install -B -DskipTests -pl testsuite/integration-arquillian/servers/auth-server/quarkus -Pauth-server-quarkus -Pdb-aurora-postgres -Dmaven.build.cache.enabled=true"
|
||||
|
||||
- name: Run Aurora migration tests on EC2
|
||||
id: aurora-migration-tests
|
||||
env:
|
||||
old-version: 24.0.4
|
||||
run: |
|
||||
EC2_CLUSTER_NAME=${{ steps.ec2-create.outputs.ec2_cluster }}
|
||||
AWS_REGION=${{ steps.aurora-init.outputs.region }}
|
||||
PROPS='${{ steps.aurora-init.outputs.maven_properties }}'
|
||||
|
||||
PROPS+=" -Dauth.server.db.host=${{ steps.aurora-create.outputs.endpoint }} -Dkeycloak.connectionsJpa.password=${{ steps.aurora-init.outputs.aurora-cluster-password }}"
|
||||
PROPS+=" -Djdbc.mvn.groupId=software.amazon.jdbc -Djdbc.mvn.artifactId=aws-advanced-jdbc-wrapper -Djdbc.mvn.version=2.3.1 -Djdbc.driver.tmp.dir=target/unpacked/keycloak-${{ env.old-version }}/providers"
|
||||
|
||||
cd .github/scripts/ansible
|
||||
export CLUSTER_NAME=keycloak_$(git rev-parse --short HEAD)
|
||||
echo "ec2_cluster=${CLUSTER_NAME}" >> $GITHUB_OUTPUT
|
||||
./aws_ec2.sh requirements
|
||||
./aws_ec2.sh create ${REGION}
|
||||
./keycloak_ec2_installer.sh ${REGION} /tmp/keycloak.zip
|
||||
./mvn_ec2_runner.sh ${REGION} "clean install -B -DskipTests -Pdistribution"
|
||||
./mvn_ec2_runner.sh ${REGION} "clean install -B -DskipTests -pl testsuite/integration-arquillian/servers/auth-server/quarkus -Pauth-server-quarkus -Pdb-aurora-postgres -Dmaven.build.cache.enabled=true"
|
||||
./mvn_ec2_runner.sh ${REGION} "test -B ${{ env.SUREFIRE_RETRY }} -Pauth-server-quarkus -Pdb-${{ matrix.db }} $PROPS -Dtest=$TESTS -pl testsuite/integration-arquillian/tests/base 2>&1 | misc/log/trimmer.sh"
|
||||
./mvn_ec2_runner.sh ${AWS_REGION} ${EC2_CLUSTER_NAME} "clean install -B ${{ env.SUREFIRE_RETRY }} -Pauth-server-quarkus -Pdb-aurora-postgres -Pauth-server-migration $PROPS -Dtest=MigrationTest -Dmigration.mode=auto -Dmigrated.auth.server.version=${{ env.old-version }} -Dmigration.import.file.name=migration-realm-${{ env.old-version }}.json -Dauth.server.ssl.required=false -f testsuite/integration-arquillian/pom.xml 2>&1 | misc/log/trimmer.sh"
|
||||
|
||||
# Copy returned surefire-report directories to workspace root to ensure they're discovered
|
||||
results=(files/keycloak/results/*)
|
||||
rsync -a $results/* ../../../
|
||||
|
||||
rm -rf $results
|
||||
|
||||
- name: Upload JVM Heapdumps
|
||||
if: always()
|
||||
uses: ./.github/actions/upload-heapdumps
|
||||
|
||||
- uses: ./.github/actions/upload-flaky-tests
|
||||
name: Upload flaky tests
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
with:
|
||||
job-name: AuroraDB IT
|
||||
|
||||
- name: Surefire reports
|
||||
if: always()
|
||||
uses: ./.github/actions/archive-surefire-reports
|
||||
with:
|
||||
job-id: migration-tests-${{ env.old-version }}-aurora-postgres
|
||||
|
||||
- name: EC2 Maven Logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: auroraDB-migration-tests-mvn-logs
|
||||
path: .github/scripts/ansible/files
|
||||
|
||||
- name: Run Aurora integration tests on EC2
|
||||
id: aurora-integration-tests
|
||||
run: |
|
||||
EC2_CLUSTER_NAME=${{ steps.ec2-create.outputs.ec2_cluster }}
|
||||
AWS_REGION=${{ steps.aurora-init.outputs.region }}
|
||||
PROPS='${{ steps.aurora-init.outputs.maven_properties }}'
|
||||
PROPS+=" -Dauth.server.db.host=${{ steps.aurora-create.outputs.endpoint }} -Dkeycloak.connectionsJpa.password=${{ steps.aurora-init.outputs.aurora-cluster-password }}"
|
||||
|
||||
TESTS=`testsuite/integration-arquillian/tests/base/testsuites/suite.sh database`
|
||||
echo "Tests: $TESTS"
|
||||
|
||||
cd .github/scripts/ansible
|
||||
./mvn_ec2_runner.sh ${AWS_REGION} ${EC2_CLUSTER_NAME} "test -B ${{ env.SUREFIRE_RETRY }} -Pauth-server-quarkus -Pdb-aurora-postgres $PROPS -Dtest=$TESTS -pl testsuite/integration-arquillian/tests/base 2>&1 | misc/log/trimmer.sh"
|
||||
|
||||
# Copy returned surefire-report directories to workspace root to ensure they're discovered
|
||||
results=(files/keycloak/results/*)
|
||||
rsync -a $results/* ../../../
|
||||
rm -rf $results
|
||||
|
||||
- name: Upload JVM Heapdumps
|
||||
if: always()
|
||||
uses: ./.github/actions/upload-heapdumps
|
||||
|
||||
- uses: ./.github/actions/upload-flaky-tests
|
||||
name: Upload flaky tests
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
with:
|
||||
job-name: AuroraDB IT
|
||||
|
||||
- name: Surefire reports
|
||||
if: always()
|
||||
uses: ./.github/actions/archive-surefire-reports
|
||||
with:
|
||||
job-id: aurora-integration-tests
|
||||
|
||||
- name: EC2 Maven Logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: aurora-integration-tests-mvn-logs
|
||||
path: .github/scripts/ansible/files
|
||||
|
||||
- name: Delete EC2 Instance
|
||||
if: always()
|
||||
working-directory: .github/scripts/ansible
|
||||
run: |
|
||||
./aws_ec2.sh delete ${{ steps.aurora-init.outputs.region }} ${{ steps.ec2-create.outputs.ec2_cluster }}
|
||||
|
||||
- name: Delete Aurora DB
|
||||
if: always()
|
||||
run: |
|
||||
gh workflow run aurora-delete.yml \
|
||||
-f name=${{ steps.aurora-init.outputs.aurora-cluster-name }} \
|
||||
-f region=${{ steps.aurora-init.outputs.region }} \
|
||||
--repo ${{ github.repository }} \
|
||||
--ref ${{ github.ref_name }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
||||
store-integration-tests:
|
||||
name: Store IT
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 75
|
||||
strategy:
|
||||
matrix:
|
||||
db: [postgres, mysql, oracle, mssql, mariadb]
|
||||
fail-fast: false
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- id: integration-test-setup
|
||||
name: Integration test setup
|
||||
uses: ./.github/actions/integration-test-setup
|
||||
|
||||
- name: Run base tests
|
||||
if: ${{ matrix.db != 'aurora-postgres' }}
|
||||
run: |
|
||||
TESTS=`testsuite/integration-arquillian/tests/base/testsuites/suite.sh database`
|
||||
echo "Tests: $TESTS"
|
||||
|
@ -521,31 +622,6 @@ jobs:
|
|||
with:
|
||||
job-id: store-integration-tests-${{ matrix.db }}
|
||||
|
||||
- name: EC2 Maven Logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: store-it-mvn-logs
|
||||
path: .github/scripts/ansible/files
|
||||
|
||||
- name: Delete Aurora EC2 Instance
|
||||
if: ${{ always() && matrix.db == 'aurora-postgres' }}
|
||||
working-directory: .github/scripts/ansible
|
||||
run: |
|
||||
export CLUSTER_NAME=${{ steps.aurora-tests.outputs.ec2_cluster }}
|
||||
./aws_ec2.sh delete ${{ steps.aurora-init.outputs.region }}
|
||||
|
||||
- name: Delete Aurora DB
|
||||
if: ${{ always() && matrix.db == 'aurora-postgres' }}
|
||||
run: |
|
||||
gh workflow run aurora-delete.yml \
|
||||
-f name=${{ steps.aurora-init.outputs.name }} \
|
||||
-f region=${{ steps.aurora-init.outputs.region }} \
|
||||
--repo ${{ github.repository }} \
|
||||
--ref ${{ github.ref_name }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
||||
store-model-tests:
|
||||
name: Store Model Tests
|
||||
runs-on: ubuntu-latest
|
||||
|
|
|
@ -83,8 +83,8 @@ public class KeycloakQuarkusServerDeployableContainer extends AbstractQuarkusDep
|
|||
ProcessBuilder pb = new ProcessBuilder(commands);
|
||||
Process p = pb.directory(wrkDir).inheritIO().start();
|
||||
try {
|
||||
if (!p.waitFor(60, TimeUnit.SECONDS)) {
|
||||
throw new IOException("Command " + command + " did not finished in 60 seconds");
|
||||
if (!p.waitFor(300, TimeUnit.SECONDS)) {
|
||||
throw new IOException("Command " + command + " did not finished in 300 seconds");
|
||||
}
|
||||
if (p.exitValue() != 0) {
|
||||
throw new IOException("Command " + command + " was executed with exit status " + p.exitValue());
|
||||
|
|
Loading…
Reference in a new issue