Commit 84c1cebf authored by Christian Ceelen's avatar Christian Ceelen
Browse files

feat: implement mutation testing with Stryker.NET

parent e071309c

README.md

+26 −0
Original line number Diff line number Diff line
@@ -226,6 +226,32 @@ In order to implement the best GitLab and SonarQube integration, the .NET templa 
  

  :information_source: the `coverlet.msbuild` package shall be added to your test projects ([see NuGet](https://www.nuget.org/packages/coverlet.msbuild))



### `dotnet-stryker` job



This job runs [stryker mutation testing](https://stryker-mutator.io/) on the configured test suits in the project. This gives you an idea on how well your tests are covering the code.



This job will post a summary of the test report and a link to the full report as a comment in an open merge request. /!\ This requires that you configure a CI/CD variable `GITLAB_TOKEN` and give it API permissions.



**Execution rules:**



- Runs a full mutation test run on release, integration, or production branches.

- Runs a differential mutation test `--since ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-${CI_DEFAULT_BRANCH}}` on MR branches.



It uses the following variables:



| Input / Variable | Description | Default value |

| ---------------- | ----------- | ------------- |

| `stryker-enabled` / `DOTNET_STRYKER_ENABLED` | Set to true to activate [Stryker](https://stryker-mutator.io/) mutation testing. | `false` |

| `stryker-opts` / `DOTNET_STRYKER_OPTS` | Addtional Stryker options controlling mutation and exit behavior. | `--break-at 0 --threshold-low 60 --threshold-high 80 -l Standard` |

| :lock: `GITLAB_TOKEN` | GitLab CLI authentication [token](https://docs.gitlab.com/cli/#authenticate-with-gitlab) | _none_ |



### Stryker Dashboard



In case you want to use the [Stryker dashboard](https://dashboard.stryker-mutator.io/) you will need to configure several things:

- Update your project files to contain _project name_ and _project version_ or use [SourceLink](https://github.com/dotnet/sourcelink#readme)

- Configure a CI/CD variable like `STRYKER_API_KEY` with the dashboard api key.

- Add `--dashboard-api-key ${STRYKER_API_KEY}` to the `stryker-opts`.



### `dotnet-format` job



This job performs code formatting validation using [`dotnet format`](https://learn.microsoft.com/en-us/dotnet/core/tools/dotnet-format) of the **whole repository**. It verifies that the code follows the formatting rules defined in `.editorconfig`.

kicker.json

+18 −0
Original line number Diff line number Diff line
@@ -72,6 +72,24 @@ 
        }

      ]

    },

    {

      "id": "stryker",

      "name": "Stryker Mutation Testing",

      "description": "Set to true to activate [Stryker](https://stryker-mutator.io/) mutation testing.",

      "enable_with": "DOTNET_STRYKER_ENABLED",

      "variables": [

        {

          "name": "DOTNET_STRYKER_OPTS",

          "description": "Addtional Stryker options controlling mutation and exit behavior.",

          "default": "--break-at 0 --threshold-low 60 --threshold-high 80 -l Standard"

        },

        {

          "name": "GITLAB_TOKEN",

          "description": "GitLab CLI authentication [token](https://docs.gitlab.com/cli/#authenticate-with-gitlab)",

          "secret": true

        }

      ]

    },

    {

      "id": "sonar",

      "name": "SonarQube",

templates/gitlab-ci-dotnet.yml

+277 −0
Original line number Diff line number Diff line
@@ -45,6 +45,13 @@ spec: 
    test-extra-args:

      description: Extra arguments used by the [dotnet test](https://learn.microsoft.com/en-us/dotnet/core/tools/dotnet-test) command

      default: ''

    stryker-enabled:

      description: Set to true to activate [Stryker](https://stryker-mutator.io/) mutation testing.

      type: boolean

      default: false

    stryker-opts:

      description: Addtional Stryker options controlling mutation exit behavior.

      default: "--break-at 0 --threshold-low 60 --threshold-high 80 -l Standard"

    sonar-host-url:

      description: SonarQube server url

      default: ''
@@ -190,6 +197,7 @@ variables: 
  DOTNET_BUILD_FILE: $[[ inputs.build-file ]]



  DOTNET_XQ_VERSION: '1.3.0'

  DOTNET_GLAB_VERSION: '1.77.0'



  # Build / test / package

  DOTNET_BUILD_ARGS: $[[ inputs.build-args ]]
@@ -203,6 +211,8 @@ variables: 


  DOTNET_TEST_DISABLED: $[[ inputs.test-disabled ]]

  DOTNET_TEST_EXTRA_ARGS: $[[ inputs.test-extra-args ]]

  DOTNET_STRYKER_ENABLED: $[[ inputs.stryker-enabled ]]

  DOTNET_STRYKER_OPTS: $[[ inputs.stryker-opts ]]



  # SonarQube

  SONAR_HOST_URL: $[[ inputs.sonar-host-url ]]
@@ -1645,6 +1655,253 @@ stages: 
    fi

  }



  function _dotnet_install_glab() {

    if [[ ! -f "${DOTNET_CLI_HOME}/bin/glab" ]]; then

      log_info "Installing GitLab CLI (glab) from GitLab"

      mkdir -p "${DOTNET_CLI_HOME}/bin"

      maybe_install_packages curl tar gzip

      local url

      url="$(_dotnet_generate_os_arch_template "https://gitlab.com/gitlab-org/cli/-/releases/v${DOTNET_GLAB_VERSION}/downloads/glab_${DOTNET_GLAB_VERSION}_<OS>_<ARCH>.tar.gz")"

      curl --location --output "${DOTNET_CLI_HOME}/bin/glab.tar.gz" "${url}"

      tar -xzf "${DOTNET_CLI_HOME}/bin/glab.tar.gz" -C "${DOTNET_CLI_HOME}" bin/glab

      rm -f "${DOTNET_CLI_HOME}/bin/glab.tar.gz"

      chmod +x "${DOTNET_CLI_HOME}/bin/glab"

    fi

  }



  function dotnet_post_comment() {

    if [[ -z "${GITLAB_TOKEN}" ]]; then

      log_warn "GITLAB_TOKEN is not configured. Cannot post comment with results in merge request."

      return

    fi

    if [[ -n "${CI_MERGE_REQUEST_IID}" ]] && [[ -n "${GITLAB_TOKEN}" ]]; then

      _dotnet_install_glab

      # shellcheck disable=SC2153

      log_info "Posting comment to Merge Request !${CI_MERGE_REQUEST_IID}."

      

      # Extract the first line to use as identifier

      local first_line

      first_line=$(echo "$1" | head -n 1)

      

      # Find and delete existing comments with the same first line

      log_info "Checking for existing comments with same identifier: ${first_line}"

      local existing_notes

      existing_notes=$(glab mr note list "$CI_MERGE_REQUEST_IID" --output json 2>/dev/null || echo "[]")

      

      if [[ -n "${existing_notes}" ]] && [[ "${existing_notes}" != "[]" ]]; then

        local note_ids

        note_ids=$(echo "${existing_notes}" | jq -r --arg first_line "${first_line}" '.[] | select(.body | startswith($first_line)) | .id')

        

        if [[ -n "${note_ids}" ]]; then

          while IFS= read -r note_id; do

            if [[ -n "${note_id}" ]]; then

              log_info "Deleting existing comment ${note_id}..."

              glab api -X DELETE "projects/${CI_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}/notes/${note_id}" 2>/dev/null || true

            fi

          done <<< "${note_ids}"

        fi

      fi

      

      glab mr note "$CI_MERGE_REQUEST_IID" --message "$1"

    else

      log_info "Skipping posting comment as no Merge Request open."

    fi

  }



  function _dotnet_convert_stryker_report() {

    TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%S")

    REPORT_URL="$3"

    maybe_install_packages jq



    jq --arg timestamp "$TIMESTAMP" \

      --arg report_url "$REPORT_URL" \

      --arg ci_job_url "${CI_JOB_URL:-}" \

    '{

      "version": "15.2.3",

      "schema": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/coverage-fuzzing-report.json",

      "scan": {

        "analyzer": {

          "id": "stryker-mutation-adapter",

          "name": "Stryker Mutation Testing Adapter",

          "url": "https://stryker-mutator.io",

          "version": "1.0.0",

          "vendor": {"name": "Custom GitLab Integration"}

        },

        "scanner": {

          "id": "stryker-net",

          "name": "Stryker.NET",

          "url": "https://stryker-mutator.io/docs/stryker-net/introduction",

          "version": "4.0+",

          "vendor": {"name": "Stryker Mutator"}

        },

        "start_time": $timestamp,

        "end_time": $timestamp,

        "status": "success",

        "type": "coverage_fuzzing"

      },

      "vulnerabilities": [

        .files | to_entries[] | 

        . as $fileEntry |

        .value.mutants[] | 

        select(.status == "Survived" or .status == "NoCoverage" or .status == "Timeout") |

        {

          "id": ($fileEntry.key + ":" + (.location.start.line | tostring) + ":" + .id),

          "category": "coverage_fuzzing",

          "name": ("Mutation " + .status + ": " + .mutatorName + " in " + ($fileEntry.key | split("/") | last)),

          "message": .mutatorName,

          "description": (

            "**Mutation " + .status + "** - Test suite failed to detect this code mutation\n\n" +

            "- **File:** `" + $fileEntry.key + "`\n" +

            "- **Location:** Line " + (.location.start.line | tostring) + ", Column " + (.location.start.column | tostring) + "\n" +

            "- **Mutation Type:** " + .mutatorName + "\n" +

            "- **Mutated To:** `" + .replacement + "`\n" +

            "- **Status:** " + .status + 

            (if .statusReason then " (" + .statusReason + ")" else "" end) + "\n" +

            "- **Covered By:** " + ((.coveredBy // []) | length | tostring) + " test(s)\n" +

            (if .status == "NoCoverage" then "\n:warning: **No test coverage** for this code path\n" else "" end) +

            (if .status == "Timeout" then "\n:warning: **Mutation caused timeout** - possible infinite loop\n" else "" end) +

            "\n[:bar_chart: View Full Mutation Report](" + $report_url + "#mutant=" + .id + ")"

          ),

          "severity": (

            if .status == "NoCoverage" then "High"

            elif .status == "Timeout" then "Medium"

            else "Medium"

            end

          ),

          "identifiers": [

            {

              "type": "stryker-mutant-id",

              "name": ("Mutant #" + .id),

              "value": .id,

              "url": ($report_url + "#mutant=" + .id)

            },

            {

              "type": "stryker-mutation-type",

              "name": .mutatorName,

              "value": .mutatorName

            }

          ],

          "links": [

            {

              "name": "View in Stryker HTML Report",

              "url": ($report_url + "#mutant=" + .id)

            }

          ],

          "location": {

            "file": $fileEntry.key,

            "start_line": .location.start.line,

            "end_line": .location.end.line,

            "crash_type": ("Mutation-" + .status),

            "crash_state": (.mutatorName + "\n" + $fileEntry.key + ":" + (.location.start.line | tostring)),

            "stacktrace_snippet": (

              "File: " + $fileEntry.key + "\n" +

              "Line: " + (.location.start.line | tostring) + "\n" +

              "Mutation: " + .mutatorName + "\n" +

              "Replacement: " + .replacement + "\n" +

              "Status: " + .status + "\n" +

              "Covered by " + ((.coveredBy // []) | length | tostring) + " test(s)"

            )

          },

          "scanner": {

            "id": "stryker-net",

            "name": "Stryker.NET"

          }

        }

      ]

    }' "$1" > "$2"

  }



  function _dotnet_build_stryker_report_url(){

    local artifact_path="reports/dotnet-stryker-${DOTNET_PROJECT_NAME}.stryker.html"

    if [[ "${DOTNET_PROJECT_DIR}" != "." ]]; then

      artifact_path="${DOTNET_PROJECT_DIR}/reports/dotnet-stryker-${DOTNET_PROJECT_NAME}.stryker.html"

    fi

    # Remove the root namespace from the project path for GitLab Pages URL

    local project_path_without_root="${CI_PROJECT_PATH#"${CI_PROJECT_ROOT_NAMESPACE}/"}"

    echo "https://${CI_PROJECT_ROOT_NAMESPACE}.gitlab.io/-/${project_path_without_root}/-/jobs/${CI_JOB_ID}/artifacts/${artifact_path}"

  }



  function _dotnet_build_stryker_report_download_url(){

    local artifact_path="reports/dotnet-stryker-${DOTNET_PROJECT_NAME}.stryker.html"

    if [[ "${DOTNET_PROJECT_DIR}" != "." ]]; then

      artifact_path="${DOTNET_PROJECT_DIR}/reports/dotnet-stryker-${DOTNET_PROJECT_NAME}.stryker.html"

    fi

    echo "${CI_JOB_URL}/artifacts/file/${artifact_path}"

  }



  function _dotnet_post_stryker_report() {

    # prepare and post the markdown report

    log_info "Posting Stryker mutation testing summary to Merge Request"

    sed -i "s|${CI_PROJECT_DIR}/||g" "${DOTNET_PROJECT_DIR}/reports/mutation-report.md"

    sed -i "s|# Mutation Testing Summary|# [Mutation Testing Summary: ${DOTNET_PROJECT_NAME}]($(_dotnet_build_stryker_report_url)) \n ([Artifact location]($(_dotnet_build_stryker_report_download_url)))|g" "${DOTNET_PROJECT_DIR}/reports/mutation-report.md"

    if [[ $(wc -l "${DOTNET_PROJECT_DIR}/reports/mutation-report.md" | cut -d' ' -f 1) -gt 40 ]]; then

      log_info "Report too long. Removing 100.00% lines."

      grep -v "100.00%" "${DOTNET_PROJECT_DIR}/reports/mutation-report.md" > "${DOTNET_PROJECT_DIR}/reports/mutation-report.tmp.md"

      mv "${DOTNET_PROJECT_DIR}/reports/mutation-report.tmp.md" "${DOTNET_PROJECT_DIR}/reports/mutation-report.md"      

    fi

    dotnet_post_comment "$(cat "${DOTNET_PROJECT_DIR}/reports/mutation-report.md")"

  }



  function _dotnet_manage_stryker_reports() {

    # Remove leading CI_PROJECT_DIR from report paths and rename and format the reports

    sed -i "s|${CI_PROJECT_DIR}/||g" "${DOTNET_PROJECT_DIR}/reports/mutation-report.html"

    mv "${DOTNET_PROJECT_DIR}/reports/mutation-report.html" "${DOTNET_PROJECT_DIR}/reports/dotnet-stryker-${DOTNET_PROJECT_NAME}.stryker.html"

    sed -i "s|${CI_PROJECT_DIR}/||g" "${DOTNET_PROJECT_DIR}/reports/mutation-report.json"

    _dotnet_convert_stryker_report "${DOTNET_PROJECT_DIR}/reports/mutation-report.json" "${DOTNET_PROJECT_DIR}/reports/dotnet-stryker-${DOTNET_PROJECT_NAME}.coverage_fuzzing.json" "$(_dotnet_build_stryker_report_url)"

    mv "${DOTNET_PROJECT_DIR}/reports/mutation-report.json" "${DOTNET_PROJECT_DIR}/reports/dotnet-stryker-${DOTNET_PROJECT_NAME}.stryker.json"

  }



  function dotnet_run_stryker() {

    log_info "Running Stryker mutation testing on project: ${DOTNET_PROJECT_DIR}/${DOTNET_BUILD_FILE}"

    dotnet_tool_install dotnet-stryker

    _dotnet_run_restore

    local stryker_config_file

    local config_opts

    if [[ -f "${DOTNET_PROJECT_DIR}/stryker-config.json" ]]; then

      stryker_config_file="${DOTNET_PROJECT_DIR}/stryker-config.json"

      config_opts="--config-file ${stryker_config_file}"

    elif [[ -f "stryker-config.json" ]]; then

      stryker_config_file="stryker-config.json"

      config_opts="--config-file ${stryker_config_file}"

    else

      stryker_config_file="${DOTNET_PROJECT_DIR}/stryker-config.json"

      echo "{}" > "${stryker_config_file}"

      config_opts=""

    fi

    local version

    version=${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-${CI_COMMIT_REF_NAME}}

    local since_opts=""

    if ! [[ $CI_COMMIT_REF_NAME =~ $RELEASE_REF || $CI_COMMIT_REF_NAME =~ $INTEG_REF || $CI_COMMIT_REF_NAME =~ $PROD_REF ]];  then

      local since_branch="${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-${CI_DEFAULT_BRANCH}}"

      git fetch origin "${since_branch}:${since_branch}"

      since_opts="--since:${since_branch}"

      maybe_install_packages jq

      jq '(. // {}) |

        .["stryker-config"] = (.["stryker-config"] // {}) |

        .["stryker-config"].since = (.["stryker-config"].since // {}) |

        .["stryker-config"].since["ignore-changes-in"] = ((.["stryker-config"].since["ignore-changes-in"] // []) + ["**/paket-packages/**", "**/.dotnetcli/**"])

      ' "${stryker_config_file}" > "${DOTNET_PROJECT_DIR}/stryker-pipeline.json"

      config_opts="--config-file ${DOTNET_PROJECT_DIR}/stryker-pipeline.json"

    fi

    

    local stryker_rc

    stryker_rc=0

    log_info "dotnet stryker -p/-s ${DOTNET_PROJECT_DIR}/${DOTNET_BUILD_FILE} ${config_opts} --skip-version-check --version ${DOTNET_PROJECT_VERSION_FULL} ${since_opts} $DOTNET_STRYKER_OPTS -r Html -r Markdown -r Dots -r Json -O ${DOTNET_PROJECT_DIR}"

    # shellcheck disable=SC2086,SC2090

    if _dotnet_is_solution_file "${DOTNET_PROJECT_DIR}/${DOTNET_BUILD_FILE}"; then

      dotnet stryker -s "${DOTNET_PROJECT_DIR}/${DOTNET_BUILD_FILE}" ${config_opts} --skip-version-check --version "${DOTNET_PROJECT_VERSION_FULL}" ${since_opts} $DOTNET_STRYKER_OPTS -r Html -r Markdown -r Dots -r Json -O "${DOTNET_PROJECT_DIR}" || stryker_rc=$?

    else

      dotnet stryker -p "${DOTNET_PROJECT_DIR}/${DOTNET_BUILD_FILE}" ${config_opts} --skip-version-check --version "${DOTNET_PROJECT_VERSION_FULL}" ${since_opts} $DOTNET_STRYKER_OPTS -r Html -r Markdown -r Dots -r Json -O "${DOTNET_PROJECT_DIR}" || stryker_rc=$?

    fi



    _dotnet_manage_stryker_reports

    _dotnet_post_stryker_report

    if [[ "${stryker_rc}" != "0" ]]; then 

      fail "Stryker mutation testing detected issues"

    fi

    log_info "Stryker mutation testing completed successfully"

  }

  

  unscope_variables

  eval_all_secrets
@@ -1705,6 +1962,26 @@ dotnet-build: 
        path: '${DOTNET_PROJECT_DIR}/reports/dotnet-test-*.cobertura.xml'

      junit: '${DOTNET_PROJECT_DIR}/reports/dotnet-test-*.junit.xml'



    

dotnet-stryker:

  extends: .dotnet-env-base

  stage: test

  needs: []

  dependencies: []

  script:

      - dotnet_run_stryker 

  artifacts:

    when: always

    paths:

      - "${DOTNET_PROJECT_DIR}/reports/dotnet-stryker-*"

    reports:

      coverage_fuzzing: "${DOTNET_PROJECT_DIR}/reports/dotnet-stryker-*.coverage_fuzzing.json"



  rules:

    - if: '$DOTNET_STRYKER_ENABLED != "true"'

      when: never

    - !reference [.test-policy, rules]



# Sonar

dotnet-sonar:

  extends: .dotnet-env-base