From a558d2f546ad99b7c71ae883f0c1bbc0a14088bb Mon Sep 17 00:00:00 2001 From: andreaTP Date: Tue, 12 Apr 2022 10:19:24 +0100 Subject: [PATCH] OLM tests using ttl.sh --- .github/workflows/operator-ci.yml | 164 ++++++++++++++---- .../manifests/clusterserviceversion.yaml | 13 +- operator/scripts/check-examples-installed.sh | 2 +- operator/scripts/create-olm-test-resources.sh | 8 +- operator/scripts/install-keycloak-operator.sh | 15 +- operator/scripts/olm-testing.sh | 32 ++++ operator/scripts/prepare-olm-test.sh | 18 +- 7 files changed, 199 insertions(+), 53 deletions(-) create mode 100755 operator/scripts/olm-testing.sh diff --git a/.github/workflows/operator-ci.yml b/.github/workflows/operator-ci.yml index f89e0e2214..5f995d5619 100644 --- a/.github/workflows/operator-ci.yml +++ b/.github/workflows/operator-ci.yml @@ -9,6 +9,8 @@ on: env: JDK_VERSION: 11 + MINIKUBE_VERSION: "v1.24.0" + KUBERNETES_VERSION: "v1.22.3" concurrency: # Only run once for latest commit per ref and cancel other (previous) runs. @@ -17,7 +19,7 @@ concurrency: jobs: build: - name: Build + name: Build distribution if: ${{ ( github.event_name != 'schedule' ) || ( github.event_name == 'schedule' && github.repository == 'keycloak/keycloak' ) }} runs-on: ubuntu-latest steps: @@ -28,60 +30,160 @@ jobs: with: distribution: 'temurin' java-version: ${{ env.JDK_VERSION }} - - name: Cache Maven packages - id: cache - uses: actions/cache@v3 - with: - path: | - ~/.m2/repository - key: cache-1-${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: cache-1-${{ runner.os }}-m2 - + cache: 'maven' - name: Create the Keycloak distribution run: | mvn clean install -Pdistribution -DskipTests -DskipExamples -DskipTestsuite + - name: Store Keycloak distribution + id: store-keycloak + uses: actions/upload-artifact@v3 + with: + name: keycloak-distribution + retention-days: 1 + path: quarkus/dist/target/keycloak*.tar.gz + test-local: + name: Test local + runs-on: ubuntu-latest + needs: [build] + steps: + - uses: actions/checkout@v3 + - name: Set outputs + id: vars + run: echo "::set-output name=version::0.0.1-${GITHUB_SHA::6}" + - name: Update maven settings + run: mkdir -p ~/.m2 ; cp .github/settings.xml ~/.m2/ + - uses: actions/setup-java@v2 + with: + distribution: 'temurin' + java-version: ${{ env.JDK_VERSION }} + cache: 'maven' - name: Setup Minikube-Kubernetes uses: manusa/actions-setup-minikube@v2.4.3 with: - minikube version: v1.24.0 - kubernetes version: v1.22.3 + minikube version: ${{ env.MINIKUBE_VERSION }} + kubernetes version: ${{ env.KUBERNETES_VERSION }} github token: ${{ secrets.GITHUB_TOKEN }} driver: docker start args: '--addons=ingress' - - - name: Build the Keycloak Docker image - run: | - cd quarkus - cp dist/target/keycloak-*.tar.gz container/ - cd container - eval $(minikube -p minikube docker-env) - docker build --build-arg KEYCLOAK_DIST=$(ls keycloak-*.tar.gz) . -t keycloak:${GITHUB_SHA} - - - name: Build a custom pre-augmented Keycloak Docker image - working-directory: operator + - name: Download keycloak distribution + id: download-keycloak-dist + uses: actions/download-artifact@v3 + with: + name: keycloak-distribution + path: quarkus/container + - name: Build Keycloak Docker images run: | eval $(minikube -p minikube docker-env) - ./scripts/build-testing-docker-images.sh ${GITHUB_SHA} keycloak custom-keycloak + (cd quarkus/container && docker build --build-arg KEYCLOAK_DIST=$(ls keycloak-*.tar.gz) . -t keycloak:${{ steps.vars.outputs.version }}) + (cd operator && ./scripts/build-testing-docker-images.sh ${{ steps.vars.outputs.version }} keycloak custom-keycloak) - name: Test operator running locally working-directory: operator run: | mvn clean verify \ -Dquarkus.kubernetes.deployment-target=kubernetes \ - -Doperator.keycloak.image=keycloak:${GITHUB_SHA} \ + -Doperator.keycloak.image=keycloak:${{ steps.vars.outputs.version }} \ + -Dtest.operator.custom.image=custom-keycloak:${{ steps.vars.outputs.version }} \ -Doperator.keycloak.image-pull-policy=Never \ - -Dtest.operator.kubernetes.ip=$(minikube ip) \ - -Dtest.operator.custom.image=custom-keycloak:${GITHUB_SHA} + -Dtest.operator.kubernetes.ip=$(minikube ip) + + test-remote: + name: Test remote + runs-on: ubuntu-latest + needs: [build] + steps: + - uses: actions/checkout@v3 + - name: Set outputs + id: vars + run: echo "::set-output name=version::0.0.1-${GITHUB_SHA::6}" + - name: Update maven settings + run: mkdir -p ~/.m2 ; cp .github/settings.xml ~/.m2/ + - uses: actions/setup-java@v2 + with: + distribution: 'temurin' + java-version: ${{ env.JDK_VERSION }} + cache: 'maven' + - name: Setup Minikube-Kubernetes + uses: manusa/actions-setup-minikube@v2.4.3 + with: + minikube version: ${{ env.MINIKUBE_VERSION }} + kubernetes version: ${{ env.KUBERNETES_VERSION }} + github token: ${{ secrets.GITHUB_TOKEN }} + driver: docker + start args: '--addons=ingress' + - name: Download keycloak distribution + id: download-keycloak-dist + uses: actions/download-artifact@v3 + with: + name: keycloak-distribution + path: quarkus/container + - name: Build Keycloak Docker images + run: | + eval $(minikube -p minikube docker-env) + (cd quarkus/container && docker build --build-arg KEYCLOAK_DIST=$(ls keycloak-*.tar.gz) . -t keycloak:${{ steps.vars.outputs.version }}) + (cd operator && ./scripts/build-testing-docker-images.sh ${{ steps.vars.outputs.version }} keycloak custom-keycloak) - name: Test operator running in cluster working-directory: operator run: | eval $(minikube -p minikube docker-env) mvn clean verify \ - -Dquarkus.container-image.build=true -Dquarkus.container-image.tag=test \ + -Dquarkus.container-image.build=true \ -Dquarkus.kubernetes.deployment-target=kubernetes \ - -Dquarkus.jib.jvm-arguments="-Djava.util.logging.manager=org.jboss.logmanager.LogManager","-Doperator.keycloak.image=keycloak:${GITHUB_SHA}","-Doperator.keycloak.image-pull-policy=Never" \ + -Dquarkus.jib.jvm-arguments="-Djava.util.logging.manager=org.jboss.logmanager.LogManager","-Doperator.keycloak.image=keycloak:${{ steps.vars.outputs.version }}",-Doperator.keycloak.image-pull-policy=Never \ + -Dtest.operator.custom.image=custom-keycloak:${{ steps.vars.outputs.version }} \ --no-transfer-progress -Dtest.operator.deployment=remote \ - -Dtest.operator.kubernetes.ip=$(minikube ip) \ - -Dtest.operator.custom.image=custom-keycloak:${GITHUB_SHA} + -Dtest.operator.kubernetes.ip=$(minikube ip) + + test-olm: + name: Test OLM installation + runs-on: ubuntu-latest + needs: [build] + steps: + - uses: actions/checkout@v3 + - name: Update maven settings + run: mkdir -p ~/.m2 ; cp .github/settings.xml ~/.m2/ + - uses: actions/setup-java@v2 + with: + distribution: 'temurin' + java-version: ${{ env.JDK_VERSION }} + cache: 'maven' + - name: Setup Minikube-Kubernetes + uses: manusa/actions-setup-minikube@v2.4.3 + with: + minikube version: ${{ env.MINIKUBE_VERSION }} + kubernetes version: ${{ env.KUBERNETES_VERSION }} + github token: ${{ secrets.GITHUB_TOKEN }} + driver: docker + - name: Install OPM + uses: redhat-actions/openshift-tools-installer@v1 + with: + source: "github" + opm: "1.21.0" + - name: Install Yq + run: sudo snap install yq + - name: Install OLM + working-directory: operator + run: ./scripts/install-olm.sh + - name: Download keycloak distribution + id: download-keycloak-dist + uses: actions/download-artifact@v3 + with: + name: keycloak-distribution + path: quarkus/container + - name: Arrange OLM test installation + working-directory: operator + run: | + eval $(minikube -p minikube docker-env) + ./scripts/olm-testing.sh ${GITHUB_SHA::6} + + - name: Deploy an example Keycloak and wait for it to be ready + working-directory: operator + run: | + kubectl apply -f src/main/resources/example-postgres.yaml + ./scripts/check-crds-installed.sh + kubectl apply -f src/main/resources/example-keycloak.yml + kubectl apply -f src/main/resources/example-realm.yaml + # Wait for the CRs to be ready + ./scripts/check-examples-installed.sh diff --git a/operator/olm-base/manifests/clusterserviceversion.yaml b/operator/olm-base/manifests/clusterserviceversion.yaml index b231be1706..f4a7954886 100644 --- a/operator/olm-base/manifests/clusterserviceversion.yaml +++ b/operator/olm-base/manifests/clusterserviceversion.yaml @@ -13,7 +13,7 @@ metadata: alm-examples: |- [ { - "apiVersion": "keycloak.org/v2alpha1", + "apiVersion": "k8s.keycloak.org/v2alpha1", "kind": "Keycloak", "metadata": { "name": "example-keycloak", @@ -28,7 +28,7 @@ metadata: } }, { - "apiVersion": "keycloak.org/v2alpha1", + "apiVersion": "k8s.keycloak.org/v2alpha1", "kind": "KeycloakRealmImport", "metadata": { "name": "example-keycloak-realm-import", @@ -51,17 +51,12 @@ spec: - description: "Represents a Keycloak Instance" displayName: "Keycloak" kind: Keycloak - name: keycloaks.keycloak.org + name: keycloaks.k8s.keycloak.org version: v2alpha1 - - description: "Deprecated - Represents a Keycloak Instance" - displayName: "Deprecated - Keycloak" - kind: Keycloak - name: keycloaks.keycloak.org - version: v1alpha1 - description: "Represents a Keycloak Realm Import" displayName: "KeycloakRealmImport" kind: KeycloakRealmImport - name: keycloakrealmimports.keycloak.org + name: keycloakrealmimports.k8s.keycloak.org version: v2alpha1 description: | A Kubernetes Operator based on the Operator SDK for installing and managing Keycloak. diff --git a/operator/scripts/check-examples-installed.sh b/operator/scripts/check-examples-installed.sh index 8cdb67bbec..de8d1f18a5 100755 --- a/operator/scripts/check-examples-installed.sh +++ b/operator/scripts/check-examples-installed.sh @@ -1,7 +1,7 @@ #! /bin/bash set -euxo pipefail -max_retries=240 +max_retries=500 c=0 while [[ $(kubectl get keycloaks/example-kc -o jsonpath="{.status.conditions[?(@.type == 'Ready')].status}") != "true" ]] do diff --git a/operator/scripts/create-olm-test-resources.sh b/operator/scripts/create-olm-test-resources.sh index 5171029248..ca52a136d1 100755 --- a/operator/scripts/create-olm-test-resources.sh +++ b/operator/scripts/create-olm-test-resources.sh @@ -4,6 +4,8 @@ set -euxo pipefail VERSION=$1 DOCKER_REGISTRY=$2 +UUID=${3:-""} + SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) rm -rf $SCRIPT_DIR/../olm/testing-resources @@ -17,12 +19,12 @@ metadata: namespace: default spec: sourceType: grpc - image: $DOCKER_REGISTRY/keycloak-test-catalog:$VERSION + image: $DOCKER_REGISTRY/${UUID}keycloak-test-catalog:$VERSION displayName: Keycloak Test Catalog publisher: Me updateStrategy: registryPoll: - interval: 10m + interval: 2m EOF cat << EOF >> $SCRIPT_DIR/../olm/testing-resources/operatorgroup.yaml @@ -51,5 +53,5 @@ spec: config: env: - name: "OPERATOR_KEYCLOAK_IMAGE" - value: "$DOCKER_REGISTRY/keycloak:$VERSION" + value: "$DOCKER_REGISTRY/${UUID}keycloak:$VERSION" EOF diff --git a/operator/scripts/install-keycloak-operator.sh b/operator/scripts/install-keycloak-operator.sh index d0f9bfe461..5e202e8497 100755 --- a/operator/scripts/install-keycloak-operator.sh +++ b/operator/scripts/install-keycloak-operator.sh @@ -6,4 +6,17 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) # Delete the default catalog if it exists sh -c "kubectl delete catalogsources operatorhubio-catalog -n olm | true" -kubectl apply -f $SCRIPT_DIR/../olm/testing-resources +kubectl apply -f $SCRIPT_DIR/../olm/testing-resources/catalog.yaml + +# Wait for the catalog to be healthy +max_retries=200 +c=0 +while [[ $(kubectl get catalogsources test-catalog -o jsonpath="{.status.connectionState.lastObservedState}") != "READY" ]] +do + echo "waiting for the test-catalog to be ready" + ((c++)) && ((c==max_retries)) && exit -1 + sleep 1 +done + +kubectl apply -f $SCRIPT_DIR/../olm/testing-resources/operatorgroup.yaml +kubectl apply -f $SCRIPT_DIR/../olm/testing-resources/subscription.yaml diff --git a/operator/scripts/olm-testing.sh b/operator/scripts/olm-testing.sh new file mode 100755 index 0000000000..a188e1b28b --- /dev/null +++ b/operator/scripts/olm-testing.sh @@ -0,0 +1,32 @@ +#! /bin/bash +set -euxo pipefail + +UUID=${1:-$(git rev-parse --short HEAD)} + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +# This version translates to one day for ttl.sh +VERSION="86400000.0.0" + +# Build Keycloak Docker image (the keycloak tar.gz should already be in the container folder) +( + cd $SCRIPT_DIR/../../quarkus/container + + docker build --build-arg KEYCLOAK_DIST=$(ls keycloak-*.tar.gz) . -t "ttl.sh/${UUID}keycloak:${VERSION}" + docker push "ttl.sh/${UUID}keycloak:${VERSION}" +) + +# Build the operator Docker image +( + cd $SCRIPT_DIR/../ + mvn clean package \ + -Dquarkus.container-image.build=true \ + -Dquarkus.container-image.image="ttl.sh/${UUID}keycloak-operator:${VERSION}" \ + -DskipTests + # JIB patching on images doesn't work reliably with ttl.sh + docker push "ttl.sh/${UUID}keycloak-operator:${VERSION}" +) + +$SCRIPT_DIR/prepare-olm-test.sh ttl.sh ${VERSION} NONE ${UUID} + +$SCRIPT_DIR/install-keycloak-operator.sh diff --git a/operator/scripts/prepare-olm-test.sh b/operator/scripts/prepare-olm-test.sh index e093517b67..5e69cbbaa9 100755 --- a/operator/scripts/prepare-olm-test.sh +++ b/operator/scripts/prepare-olm-test.sh @@ -8,25 +8,27 @@ DOCKER_REGISTRY="$1" VERSION="$2" PREV_VERSION="$3" +UUID=${4:-""} + OPERATOR_IMAGE_NAME="keycloak-operator" -OPERATOR_DOCKER_IMAGE="$DOCKER_REGISTRY/$OPERATOR_IMAGE_NAME" +OPERATOR_DOCKER_IMAGE="$DOCKER_REGISTRY/${UUID}$OPERATOR_IMAGE_NAME" # Create OLM bundle $SCRIPT_DIR/create-olm-bundle.sh $VERSION $PREV_VERSION $OPERATOR_DOCKER_IMAGE (cd $SCRIPT_DIR/../olm/$VERSION && \ - docker build --label "quay.expires-after=20h" -t $DOCKER_REGISTRY/keycloak-operator-bundle:$VERSION -f bundle.Dockerfile . && \ - docker push $DOCKER_REGISTRY/keycloak-operator-bundle:$VERSION) + docker build -t $DOCKER_REGISTRY/${UUID}keycloak-operator-bundle:$VERSION -f bundle.Dockerfile . && \ + docker push $DOCKER_REGISTRY/${UUID}keycloak-operator-bundle:$VERSION) # Verify the bundle -opm alpha bundle validate --tag $DOCKER_REGISTRY/keycloak-operator-bundle:$VERSION --image-builder docker +opm alpha bundle validate --tag $DOCKER_REGISTRY/${UUID}keycloak-operator-bundle:$VERSION --image-builder docker # Create the test-catalog -$SCRIPT_DIR/create-olm-test-catalog.sh $VERSION $DOCKER_REGISTRY/keycloak-operator-bundle +$SCRIPT_DIR/create-olm-test-catalog.sh $VERSION $DOCKER_REGISTRY/${UUID}keycloak-operator-bundle (cd $SCRIPT_DIR/../olm/catalog && \ - docker build --label "quay.expires-after=20h" -f test-catalog.Dockerfile -t $DOCKER_REGISTRY/keycloak-test-catalog:$VERSION . && \ - docker push $DOCKER_REGISTRY/keycloak-test-catalog:$VERSION) + docker build -f test-catalog.Dockerfile -t $DOCKER_REGISTRY/${UUID}keycloak-test-catalog:$VERSION . && \ + docker push $DOCKER_REGISTRY/${UUID}keycloak-test-catalog:$VERSION) # Create testing resources -$SCRIPT_DIR/create-olm-test-resources.sh $VERSION $DOCKER_REGISTRY +$SCRIPT_DIR/create-olm-test-resources.sh $VERSION $DOCKER_REGISTRY ${UUID}