Normalize JVM heap usage in tests and handle OOM situations

Closes #16089

.github/actions/upload-heapdumps/action.yml

@@ -0,0 +1,12 @@
+name: Upload JVM Heapdumps
+description: Upload JVM Heapdumps
+
+runs:
+  using: composite
+  steps:
+    - id: upload-jvm-heapdumps
+      name: Upload JVM Heapdumps
+      uses: actions/upload-artifact@v3
+      with:
+        name: jvm-heap-dumps
+        path: '**/java_pid*.hprof'

.github/workflows/ci.yml

@@ -58,6 +58,10 @@ jobs:
           
           ./mvnw test -nsu -B -pl "$PROJECTS" -am
 
+      - name: Upload JVM Heapdumps
+        if: always()
+        uses: ./.github/actions/upload-heapdumps
+
       - id: upload-surefire-reports
         name: Upload Surefire reports
         uses: ./.github/actions/upload-surefire-reports
@@ -88,6 +92,10 @@ jobs:
           echo "Tests: $TESTS"
           ./mvnw test -Dsurefire.rerunFailingTestsCount=${{ env.SUREFIRE_RERUN_FAILING_COUNT }} -nsu -B -Pauth-server-quarkus -Dtest=$TESTS -pl testsuite/integration-arquillian/tests/base | misc/log/trimmer.sh
 
+      - name: Upload JVM Heapdumps
+        if: always()
+        uses: ./.github/actions/upload-heapdumps
+
       - id: upload-surefire-reports
         name: Upload Surefire reports
         uses: ./.github/actions/upload-surefire-reports
@@ -128,6 +136,10 @@ jobs:
           ./mvnw install -nsu -B -pl quarkus/tests/integration -am -DskipTests
           ./mvnw test -nsu -B -pl quarkus/tests/integration ${PARAMS["${{ matrix.server }}"]} | misc/log/trimmer.sh
 
+      - name: Upload JVM Heapdumps
+        if: always()
+        uses: ./.github/actions/upload-heapdumps
+
       - id: upload-surefire-reports
         name: Upload Surefire reports
         uses: ./.github/actions/upload-surefire-reports
@@ -165,6 +177,10 @@ jobs:
           echo "Tests: $TESTS"
           ./mvnw test -nsu -B -Pauth-server-quarkus -Dtest=$TESTS -pl testsuite/integration-arquillian/tests/base | misc/log/trimmer.sh
 
+      - name: Upload JVM Heapdumps
+        if: always()
+        uses: ./.github/actions/upload-heapdumps
+
       - id: upload-surefire-reports
         name: Upload Surefire reports
         uses: ./.github/actions/upload-surefire-reports
@@ -218,6 +234,10 @@ jobs:
           echo "Tests: $TESTS"
           ./mvnw test -nsu -B -Pauth-server-quarkus ${PARAMS["${{ matrix.db }}"]} -Dtest=$TESTS -pl testsuite/integration-arquillian/tests/base | misc/log/trimmer.sh
 
+      - name: Upload JVM Heapdumps
+        if: always()
+        uses: ./.github/actions/upload-heapdumps
+
       - id: upload-surefire-reports
         name: Upload Surefire reports
         uses: ./.github/actions/upload-surefire-reports
@@ -248,6 +268,10 @@ jobs:
           echo "Tests: $TESTS"
           ./mvnw test -nsu -B -Pauth-server-quarkus -Pdb-${{ matrix.db }} -Dtest=$TESTS -pl testsuite/integration-arquillian/tests/base | misc/log/trimmer.sh
 
+      - name: Upload JVM Heapdumps
+        if: always()
+        uses: ./.github/actions/upload-heapdumps
+
       - id: upload-surefire-reports
         name: Upload Surefire reports
         uses: ./.github/actions/upload-surefire-reports
@@ -271,6 +295,10 @@ jobs:
       - name: Run model tests
         run: testsuite/model/test-all-profiles.sh
 
+      - name: Upload JVM Heapdumps
+        if: always()
+        uses: ./.github/actions/upload-heapdumps
+
   clustering-integration-tests:
     name: Legacy Clustering IT
     needs: build
@@ -289,6 +317,10 @@ jobs:
         run: |
           ./mvnw test -nsu -B -Pauth-server-cluster-quarkus -Dsession.cache.owners=2 -Dtest=**.cluster.** -pl testsuite/integration-arquillian/tests/base  | misc/log/trimmer.sh
 
+      - name: Upload JVM Heapdumps
+        if: always()
+        uses: ./.github/actions/upload-heapdumps
+
       - id: upload-surefire-reports
         name: Upload Surefire reports
         uses: ./.github/actions/upload-surefire-reports
@@ -315,6 +347,10 @@ jobs:
       - name: Run crypto tests (BCFIPS approved mode)
         run: ./mvnw test -nsu -B -am -pl crypto/default,crypto/fips1402,crypto/elytron -Dcom.redhat.fips=true -Dorg.bouncycastle.fips.approved_only=true
 
+      - name: Upload JVM Heapdumps
+        if: always()
+        uses: ./.github/actions/upload-heapdumps
+
       - id: upload-surefire-reports
         name: Upload Surefire reports
         uses: ./.github/actions/upload-surefire-reports
@@ -344,6 +380,10 @@ jobs:
           echo "Tests: $TESTS"
           ./mvnw test -nsu -B -Pauth-server-quarkus,auth-server-fips140-2 -Dtest=$TESTS -pl testsuite/integration-arquillian/tests/base | misc/log/trimmer.sh
 
+      - name: Upload JVM Heapdumps
+        if: always()
+        uses: ./.github/actions/upload-heapdumps
+
       - id: upload-surefire-reports
         name: Upload Surefire reports
         uses: ./.github/actions/upload-surefire-reports

pom.xml

@@ -1783,7 +1783,7 @@
                     <version>${surefire-plugin.version}</version>
                     <configuration>
                         <forkMode>once</forkMode>
-                        <argLine>-Djava.awt.headless=true ${surefire.memory.settings} ${surefire.system.args} -Duser.language=en -Duser.region=US</argLine>
+                        <argLine>-Djava.awt.headless=true ${surefire.memory.settings} ${surefire.system.args} -Duser.language=en -Duser.region=US -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError</argLine>
                         <runOrder>alphabetical</runOrder>
                         <failIfNoSpecifiedTests>false</failIfNoSpecifiedTests>
                     </configuration>

quarkus/tests/integration/pom.xml

@@ -108,7 +108,7 @@
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-surefire-plugin</artifactId>
                 <configuration>
-                    <argLine>-Djdk.net.hosts.file=${project.build.testOutputDirectory}/hosts_file</argLine>
+                    <argLine>-Djdk.net.hosts.file=${project.build.testOutputDirectory}/hosts_file -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError</argLine>
                     <systemPropertyVariables>
                         <kc.quarkus.tests.dist>${kc.quarkus.tests.dist}</kc.quarkus.tests.dist>
                     </systemPropertyVariables>
@@ -162,7 +162,7 @@
                         <groupId>org.apache.maven.plugins</groupId>
                         <artifactId>maven-surefire-plugin</artifactId>
                         <configuration>
-                            <argLine>-Xmx1024m -XX:MaxMetaspaceSize=512m</argLine>
+                            <argLine>-Xmx1024m -XX:MaxMetaspaceSize=512m -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError</argLine>
                             <systemPropertyVariables>
                                 <kc.test.storage.database>true</kc.test.storage.database>
                                 <!--DB Container -->

saml-core/pom.xml

@@ -104,7 +104,7 @@
                         <id>with-security-manager-all-permissions</id>
                         <goals><goal>test</goal></goals>
                         <configuration>
-                            <argLine>-Dmaven.basedir=${basedir} -Djava.security.manager -Djava.security.policy=${basedir}/src/test/resources/all-permissions.policy</argLine>
+                            <argLine>-Dmaven.basedir=${basedir} -Djava.security.manager -Djava.security.policy=${basedir}/src/test/resources/all-permissions.policy -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError</argLine>
                             <test>SecurityActionsTest</test>
                             <skip>${skip.security-manager.tests}</skip>
                         </configuration>
@@ -113,7 +113,7 @@
                         <id>with-security-manager-named-permissions</id>
                         <goals><goal>test</goal></goals>
                         <configuration>
-                            <argLine>-Djava.security.debug=access -Dmaven.basedir=${basedir} -Djava.security.manager -Djava.security.policy=${basedir}/src/test/resources/named-permissions.policy</argLine>
+                            <argLine>-Djava.security.debug=access -Dmaven.basedir=${basedir} -Djava.security.manager -Djava.security.policy=${basedir}/src/test/resources/named-permissions.policy -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError</argLine>
                             <test>SecurityActionsTest</test>
                             <skip>${skip.security-manager.tests}</skip>
                         </configuration>                        

server-spi-private/pom.xml

@@ -92,7 +92,8 @@
                         <exclude/>
                     </excludes>
                     <!-- Surefire misses to pass on correct encoding to forked JVM, even though encoding is set correct. -->
-                    <argLine>-Dfile.encoding=UTF-8</argLine>
+                    <!-- https://issues.apache.org/jira/browse/SUREFIRE-951 -->
+                    <argLine>-Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError</argLine>
                 </configuration>
             </plugin>
         </plugins>

testsuite/model/pom.xml

@@ -160,7 +160,7 @@
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-surefire-plugin</artifactId>
                 <configuration>
-                    <argLine>@{argLine}</argLine>
+                    <argLine>@{argLine} -Xmx1024m -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError</argLine>
                     <systemPropertyVariables>
                         <!-- keycloak.model.parameters lists parameter classes from 
                              org.keycloak.model.parameters package and determine enabled providers -->