Flaky test reporter (#16034)

* Flaky test reporter

* Update .github/actions/upload-flaky-tests/action.yml

.github/actions/upload-flaky-tests/action.yml

@@ -0,0 +1,42 @@
+name: Report flaky test
+description: Finds flaky tests and creates issues
+
+runs:
+  using: composite
+  steps:
+    - id: flaky-tests
+      name: Find flaky tests
+      if: github.repository == 'keycloak/keycloak'
+      shell: bash
+      run: |
+        REPO="${{ github.repository }}"
+        ISSUES_LINK="https://github.com/${{ github.repository }}/issues/"
+        PR="${{ github.event.pull_request.number }}"
+        RUN="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+        for dir in $(find -type d -name surefire-reports); do
+          for i in $(ls $dir/*.txt); do
+            if ( grep '<<< FAILURE!' $i &>/dev/null ); then
+              TEST="$(echo $i | sed 's/.txt//g' | sed 's/.*\.//g')"
+              LOG="$(cat $i)"
+              
+              TITLE="Flaky test: $TEST"
+              BODY="$RUN"$'\n''```'$'\n'"$LOG"$'\n''```'
+              ISSUE="$(gh issue list --search "$TITLE in:title" --json number --jq .[].number)"
+
+              if [ "$PR" == "" ]; then        
+                if [ "$ISSUE" == "" ]; then
+                  LINK="$(gh issue create -t "$TITLE" -b "$BODY" -l "kind/bug,area/ci,flaky-test" | grep $ISSUES_LINK)"
+                  ISSUE="$(echo $LINK | sed "s|$ISSUES_LINK||")"
+                else
+                  LINK="$(gh issue comment "$ISSUE" -b "$BODY")"
+                fi    
+              fi
+              
+              echo "## $TITLE" >> $GITHUB_STEP_SUMMARY
+              echo "$LINK" >> $GITHUB_STEP_SUMMARY
+              echo '```' >> $GITHUB_STEP_SUMMARY
+              echo "$LOG" >> $GITHUB_STEP_SUMMARY
+              echo '```' >> $GITHUB_STEP_SUMMARY
+            fi
+          done
+        done

.github/workflows/ci.yml

@@ -13,6 +13,7 @@ on:
 env:
   DEFAULT_JDK_VERSION: 11
   DEFAULT_JDK_DIST: temurin
+  SUREFIRE_RERUN_FAILING_COUNT: 2
 
 concurrency:
   # Only cancel jobs for PR updates
@@ -77,7 +78,7 @@ jobs:
         run: |
           TESTS=`testsuite/integration-arquillian/tests/base/testsuites/base-suite.sh ${{ matrix.group }}`
           echo "Tests: $TESTS"
-          ./mvnw install -nsu -B -Pauth-server-quarkus -Dtest=$TESTS -pl testsuite/integration-arquillian/tests/base | misc/log/trimmer.sh
+          ./mvnw install -Dsurefire.rerunFailingTestsCount=${{ env.SUREFIRE_RERUN_FAILING_COUNT }} -nsu -B -Pauth-server-quarkus -Dtest=$TESTS -pl testsuite/integration-arquillian/tests/base | misc/log/trimmer.sh
 
       - id: upload-surefire-reports
         name: Upload Surefire reports
@@ -87,6 +88,11 @@ jobs:
           api-key: ${{ secrets.FORESIGHT_API_KEY }}
           surefire-reports-path: 'testsuite/integration-arquillian/tests/base/target/surefire-reports/*.xml'
 
+      - uses: ./.github/actions/upload-flaky-tests
+        name: Upload flaky tests
+        env:
+          GH_TOKEN: ${{ github.token }}
+
   quarkus-integration-tests:
     name: Quarkus IT
     needs: build