From f06ba05405d9dea1954e8bf60855353abe5f29a9 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira da Silva Date: Fri, 11 Mar 2022 09:55:17 -0300 Subject: [PATCH] The CodeQL analysis is broken due to the large content of the SARIF file (#10606) The issue was originally caused by high number of flows paths per alert generated by the LDAP federation module. That was identified taking the SARIF file generated and running: ``` jq '.runs[0].results | map({query_id: .rule.id, numPaths: .codeFlows | length})' java.sarif ``` Together we reduced the number of flows paths, adding optimizations to skip some paths and avoid false alerts. Co-authored-by: Bruno Oliveira da Silva Closes #10203 Co-authored-by: Joshua Mulliken --- .github/scripts/codeql/codeql-analyze.sh | 40 +++++++++++ .../scripts/codeql/codeql-database-create.sh | 25 +++++++ .github/scripts/codeql/codeql-install.sh | 6 ++ .github/workflows/codeql-analysis.yml | 70 ++++++++++--------- 4 files changed, 107 insertions(+), 34 deletions(-) create mode 100755 .github/scripts/codeql/codeql-analyze.sh create mode 100755 .github/scripts/codeql/codeql-database-create.sh create mode 100755 .github/scripts/codeql/codeql-install.sh diff --git a/.github/scripts/codeql/codeql-analyze.sh b/.github/scripts/codeql/codeql-analyze.sh new file mode 100755 index 0000000000..fd1b399cdb --- /dev/null +++ b/.github/scripts/codeql/codeql-analyze.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +CODEQL_BINARY="./codeql/codeql" + +# Check if the binary exists +if [ ! -f "$CODEQL_BINARY" ]; +then + printf "CodeQL binary not found!" + exit 1 +fi + +upload_results () { + echo "Uploading $1" + $CODEQL_BINARY github upload-results --sarif="$1" --repository="$GITHUB_REPOSITORY" --ref="$GITHUB_REF" +} + + +# Create the database based on the specifics per language +if [ "$1" = "java" ]; +then + printf "Analyzing CodeQL Java database" + $CODEQL_BINARY database analyze "$1-database" codeql/java-queries --format=sarifv2.1.0 --output="$1".sarif --download --max-paths=1 --sarif-add-query-help + < java.sarif jq 'del(.runs[].results[].codeFlows)' > processed-java.sarif + upload_results processed-java.sarif + +elif [ "$1" = "javascript" ]; +then + printf "Analyzing themes database" + $CODEQL_BINARY database analyze themes-database codeql/javascript-queries --format=sarifv2.1.0 --output=themes.sarif --download --max-paths=1 --sarif-add-query-help + < themes.sarif jq 'del(.runs[].results[].codeFlows)' > processed-themes.sarif + upload_results processed-themes.sarif + + printf "Analyzing js-adapter database" + $CODEQL_BINARY database analyze js-adapter-database codeql/javascript-queries --format=sarifv2.1.0 --output=js-adapter.sarif --download --max-paths=1 --sarif-add-query-help + < js-adapter.sarif jq 'del(.runs[].results[].codeFlows)' > processed-js-adapter.sarif + upload_results processed-js-adapter.sarif + +fi + + diff --git a/.github/scripts/codeql/codeql-database-create.sh b/.github/scripts/codeql/codeql-database-create.sh new file mode 100755 index 0000000000..16075915ef --- /dev/null +++ b/.github/scripts/codeql/codeql-database-create.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +CODEQL_BINARY="./codeql/codeql" + +# Check if the binary exists +if [ ! -f "$CODEQL_BINARY" ]; +then + printf "CodeQL binary not found!" + exit 1 +fi + +# Create the database based on the specifics per language +if [ "$1" = "java" ]; +then + printf "Creating CodeQL Java database" + $CODEQL_BINARY database create "$1-database" --no-run-unnecessary-builds --language="$1" --command='mvn clean install -Dmaven.test.skip -DskipQuarkus -DskipTestsuite -DskipExamples -DskipTests' +elif [ "$1" = "javascript" ]; +then + printf "Creating themes database" + $CODEQL_BINARY database create themes-database --no-run-unnecessary-builds --language=javascript --source-root=themes/ --command='mvn install -Dmaven.test.skip -DskipQuarkus -DskipTestsuite -DskipExamples -DskipTests' + printf "Creating js-adapter database" + $CODEQL_BINARY database create js-adapter-database --no-run-unnecessary-builds --language=javascript --source-root=adapters/oidc/js/ --command='mvn install -Dmaven.test.skip -DskipQuarkus -DskipTestsuite -DskipExamples -DskipTests' +fi + + diff --git a/.github/scripts/codeql/codeql-install.sh b/.github/scripts/codeql/codeql-install.sh new file mode 100755 index 0000000000..4eb89b319b --- /dev/null +++ b/.github/scripts/codeql/codeql-install.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +LATEST_RELEASE_URL=$(curl -s https://api.github.com/repos/github/codeql-cli-binaries/releases/latest | grep browser_download_url | cut -d '"' -f 4 | grep -i linux) + +wget -q --show-progress "$LATEST_RELEASE_URL" +unzip codeql-linux64.zip diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 5a7bddeb6c..3ecb2eda96 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -7,17 +7,19 @@ name: "CodeQL" on: # Disable for push and pull_request until https://github.com/keycloak/keycloak/issues/10203 is resolved -# push: -# branches: [main] -# pull_request: + push: + branches: [main] + pull_request: # The branches below must be a subset of the branches above -# branches: [main] -# paths-ignore: -# - 'testsuite/**' -# - 'examples/**' -# - 'quarkus/tests/**' + branches: [main] schedule: - cron: '0 9 * * 2' + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +permissions: + security-events: write jobs: analyze: @@ -37,34 +39,34 @@ jobs: - name: Checkout repository uses: actions/checkout@v2 + - name: Install CodeQL + run: ${GITHUB_WORKSPACE}/.github/scripts/codeql/codeql-install.sh + + - uses: actions/setup-java@v2 + if: ${{ matrix.language == 'java' }} + with: + distribution: 'temurin' + java-version: '11' + - name: Update maven settings run: mkdir -p ~/.m2 ; cp .github/settings.xml ~/.m2/ - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main + # Create the codeql database for Java + - name: Create CodeQL Java database + if: ${{ matrix.language == 'java' }} + run: ${GITHUB_WORKSPACE}/.github/scripts/codeql/codeql-database-create.sh java - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v1 + # Run the analysis for Java + - name: Run CodeQL analysis for Java + if: ${{ matrix.language == 'java' }} + run: ${GITHUB_WORKSPACE}/.github/scripts/codeql/codeql-analyze.sh java + + # Create the codeql database for JavaScript + - name: Create CodeQL JavaScript database + if: ${{ matrix.language == 'javascript' }} + run: ${GITHUB_WORKSPACE}/.github/scripts/codeql/codeql-database-create.sh javascript - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + # Run the analysis for JavaScript + - name: Run CodeQL analysis for JavaScript + if: ${{ matrix.language == 'javascript' }} + run: ${GITHUB_WORKSPACE}/.github/scripts/codeql/codeql-analyze.sh javascript