Flaky test reporter (#16034)

* Flaky test reporter

* Update .github/actions/upload-flaky-tests/action.yml
This commit is contained in:
Stian Thorgersen 2022-12-16 13:02:05 +01:00 committed by GitHub
parent 220590d241
commit 60f1162300
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 49 additions and 1 deletions

View file

@ -0,0 +1,42 @@
name: Report flaky test
description: Finds flaky tests and creates issues
runs:
using: composite
steps:
- id: flaky-tests
name: Find flaky tests
if: github.repository == 'keycloak/keycloak'
shell: bash
run: |
REPO="${{ github.repository }}"
ISSUES_LINK="https://github.com/${{ github.repository }}/issues/"
PR="${{ github.event.pull_request.number }}"
RUN="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
for dir in $(find -type d -name surefire-reports); do
for i in $(ls $dir/*.txt); do
if ( grep '<<< FAILURE!' $i &>/dev/null ); then
TEST="$(echo $i | sed 's/.txt//g' | sed 's/.*\.//g')"
LOG="$(cat $i)"
TITLE="Flaky test: $TEST"
BODY="$RUN"$'\n''```'$'\n'"$LOG"$'\n''```'
ISSUE="$(gh issue list --search "$TITLE in:title" --json number --jq .[].number)"
if [ "$PR" == "" ]; then
if [ "$ISSUE" == "" ]; then
LINK="$(gh issue create -t "$TITLE" -b "$BODY" -l "kind/bug,area/ci,flaky-test" | grep $ISSUES_LINK)"
ISSUE="$(echo $LINK | sed "s|$ISSUES_LINK||")"
else
LINK="$(gh issue comment "$ISSUE" -b "$BODY")"
fi
fi
echo "## $TITLE" >> $GITHUB_STEP_SUMMARY
echo "$LINK" >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
echo "$LOG" >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
fi
done
done

View file

@ -13,6 +13,7 @@ on:
env:
DEFAULT_JDK_VERSION: 11
DEFAULT_JDK_DIST: temurin
SUREFIRE_RERUN_FAILING_COUNT: 2
concurrency:
# Only cancel jobs for PR updates
@ -77,7 +78,7 @@ jobs:
run: |
TESTS=`testsuite/integration-arquillian/tests/base/testsuites/base-suite.sh ${{ matrix.group }}`
echo "Tests: $TESTS"
./mvnw install -nsu -B -Pauth-server-quarkus -Dtest=$TESTS -pl testsuite/integration-arquillian/tests/base | misc/log/trimmer.sh
./mvnw install -Dsurefire.rerunFailingTestsCount=${{ env.SUREFIRE_RERUN_FAILING_COUNT }} -nsu -B -Pauth-server-quarkus -Dtest=$TESTS -pl testsuite/integration-arquillian/tests/base | misc/log/trimmer.sh
- id: upload-surefire-reports
name: Upload Surefire reports
@ -87,6 +88,11 @@ jobs:
api-key: ${{ secrets.FORESIGHT_API_KEY }}
surefire-reports-path: 'testsuite/integration-arquillian/tests/base/target/surefire-reports/*.xml'
- uses: ./.github/actions/upload-flaky-tests
name: Upload flaky tests
env:
GH_TOKEN: ${{ github.token }}
quarkus-integration-tests:
name: Quarkus IT
needs: build