From 373320ad48689ff4f24e0c25f30809c85e90e712 Mon Sep 17 00:00:00 2001 From: chubnastia Date: Mon, 10 Aug 2026 16:13:33 +0200 Subject: [PATCH 1/6] ci: add [fast_pass] and [only: ] options for selective pipeline execution Motivation: Currently each testing pipeline starts with 30 jobs and 45 minutes average runtime, which can be inconvenient and wasteful for targeted testing. Modification: - Introduced [fast_pass] as a new commit-message trigger to skip all unit test and coverage jobs (via .fast_pass_rules) - Added `.selective_test_base` and YAML `!reference` rules across integration test jobs: - If no `[only: ...]` tag is present, all integration test jobs run by default. - If `[only: ]` is specified, jobs evaluate word boundaries (`\b\b`) to execute only matching test suites. - Supported suite tags: `grid`, `frontend`, `xroot`, `webdav`, `nfs`, `oidc`. Result: We can now: - Skip unit tests and coverage: `[fast_pass]` - Run specific integration suites: `[only: frontend]` or `[only: nfs, grid]` - Combine both options for targeted testing: `[fast_pass] [only: webdav]` - Unmodified default pipeline execution if no triggers are present. - Reduce CI resource usage and speeds up validation for focused changes. Target: master Require-book: no Require-notes: no --- .gitlab-ci.yml | 63 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a415cab257e..2c6b358a556 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -115,15 +115,23 @@ default: rules: - if: $CI_COMMIT_TAG -# rules for running coverage -.no_release_rules: +# Rules for running coverage & skipping unit tests via [fast_pass] +.fast_pass_rules: rules: + - if: '$CI_COMMIT_MESSAGE =~ /\[\s*fast[ _]?pass\s*\]/i' + when: never - if: $CI_COMMIT_TAG == null +# Base rule for testing stage jobs: runs by default if no [only: ...] filter is present +.selective_test_base: + rules: + - if: '$CI_COMMIT_MESSAGE !~ /\[\s*only\s*:/i' + when: on_success + # cache and image configuration for jacoco report jobs .jacoco_base: image: eclipse-temurin:21-jre - extends: .no_release_rules + extends: .fast_pass_rules cache: key: "jacoco-cli-${JACOCO_VERSION}" paths: @@ -172,7 +180,7 @@ run_ut_with_coverage: image: maven:3.9.12-eclipse-temurin-21 extends: - .build_cache - - .no_release_rules + - .fast_pass_rules script: - mvn $MAVEN_CLI_OPTS -P code-coverage clean verify -DskipTests=false -Drun.slow.tests artifacts: @@ -186,7 +194,7 @@ run_ut_with_coverage: - "**/target/classes/" expire_in: 1 day -# run unit tests without coverage +# run unit tests without coverage for release pipeline run_unit_tests: stage: build image: maven:3.9.12-eclipse-temurin-21 @@ -632,13 +640,15 @@ Extract Coverage: stage: testenv_post extends: - .kubernetes_image - - .no_release_rules + - .fast_pass_rules dependencies: - get_jacoco_cli needs: - job: get_jacoco_cli artifacts: true + optional: true - job: NFS4.x protocol compliance tests + optional: true script: - JACOCO_CLI_PATH="jacoco-${JACOCO_VERSION}/lib/jacococli.jar" - PODS=$(kubectl -n $K8S_NAMESPACE get pods -l app.kubernetes.io/instance=store -o jsonpath='{.items[*].metadata.name}') @@ -673,6 +683,11 @@ Extract Coverage: Grid EL9 WN tests: stage: testing extends: .kubernetes_image + rules: + - if: '$CI_COMMIT_MESSAGE =~ /\[\s*only\s*:[^\]]*\bgrid\b[^\]]*\]/i' + when: on_success + - !reference [.selective_test_base, rules] + - when: never script: - kubectl -n $K8S_NAMESPACE apply -f .ci/wn-with-cvmfs.yaml - while ! kubectl -n $K8S_NAMESPACE wait --for=condition=Ready pod grid-tester; do sleep 1; done @@ -688,6 +703,11 @@ Grid EL9 WN tests: Frontend test suite: stage: testing extends: .kubernetes_image + rules: + - if: '$CI_COMMIT_MESSAGE =~ /\[\s*only\s*:[^\]]*\bfrontend\b[^\]]*\]/i' + when: on_success + - !reference [.selective_test_base, rules] + - when: never script: - kubectl -n $K8S_NAMESPACE apply -f .ci/frontend.yaml - while ! kubectl -n $K8S_NAMESPACE wait --for=condition=Ready pod http-tester; do sleep 1; done @@ -710,6 +730,11 @@ Frontend test suite: gsi_xroot_tests: stage: testing extends: .kubernetes_image + rules: + - if: '$CI_COMMIT_MESSAGE =~ /\[\s*only\s*:[^\]]*\bxroot\b[^\]]*\]/i' + when: on_success + - !reference [.selective_test_base, rules] + - when: never script: - kubectl -n $K8S_NAMESPACE apply -f .ci/wn-with-cvmfs-xroot.yaml - while ! kubectl -n $K8S_NAMESPACE wait --for=condition=Ready pod xroot-tester; do sleep 1; done @@ -720,6 +745,11 @@ gsi_xroot_tests: webdav_with_x509_tests: stage: testing extends: .kubernetes_image + rules: + - if: '$CI_COMMIT_MESSAGE =~ /\[\s*only\s*:[^\]]*\bwebdav\b[^\]]*\]/i' + when: on_success + - !reference [.selective_test_base, rules] + - when: never script: - kubectl -n $K8S_NAMESPACE apply -f .ci/webdav-wn-cvmfs.yaml - while ! kubectl -n $K8S_NAMESPACE wait --for=condition=Ready pod webdav-tester; do sleep 1; done @@ -730,6 +760,11 @@ webdav_with_x509_tests: NFS4.x protocol compliance tests: stage: testing extends: .kubernetes_image + rules: + - if: '$CI_COMMIT_MESSAGE =~ /\[\s*only\s*:[^\]]*\bnfs\b[^\]]*\]/i' + when: on_success + - !reference [.selective_test_base, rules] + - when: never script: - kubectl -n $K8S_NAMESPACE run pynfs-tester --image=dcache/pynfs:0.5 --restart=Never --command -- sleep 3600 - while ! kubectl -n $K8S_NAMESPACE wait --for=condition=Ready pod pynfs-tester; do sleep 1; done @@ -777,6 +812,11 @@ NFS4.x protocol compliance tests: Run OIDC test: stage: testing extends: .kubernetes_image + rules: + - if: '$CI_COMMIT_MESSAGE =~ /\[\s*only\s*:[^\]]*\boidc\b[^\]]*\]/i' + when: on_success + - !reference [.selective_test_base, rules] + - when: never script: - kubectl -n $K8S_NAMESPACE run oidc-tester --image=almalinux:9 --restart=Never --command -- sleep 3600 - while ! kubectl -n $K8S_NAMESPACE wait --for=condition=Ready pod oidc-tester; do sleep 1; done @@ -786,7 +826,9 @@ Run OIDC test: #Job to find JaCoCo CLI in cache or download it get_jacoco_cli: stage: build - extends: .jacoco_base + extends: + - .jacoco_base + - .fast_pass_rules cache: key: "jacoco-cli-${JACOCO_VERSION}" paths: @@ -804,13 +846,18 @@ get_jacoco_cli: # Job to generate JaCoCo report using script generate_jacoco_report: stage: testenv_post - extends: .jacoco_base + extends: + - .jacoco_base + - .fast_pass_rules needs: - job: run_ut_with_coverage artifacts: true + optional: true - job: get_jacoco_cli + optional: true - job: Extract Coverage artifacts: true + optional: true script: - ./generate-jacoco-report.sh artifacts: From a8fcabd637d332ebee42e42805ae28466ce423bf Mon Sep 17 00:00:00 2001 From: chubnastia Date: Tue, 8 Sep 2026 14:40:53 +0200 Subject: [PATCH 2/6] trial [fast pass] [only: nfs] --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2c6b358a556..284621aa5b1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -82,6 +82,8 @@ default: - kubernetes - dcache-dev dependencies: [] + rules: + - where: always .kubernetes_image: extends: .kubernetes_tags From 5d4df2ef87eb33f4eef0cedef8c21647552c218e Mon Sep 17 00:00:00 2001 From: chubnastia Date: Tue, 8 Sep 2026 16:59:42 +0200 Subject: [PATCH 3/6] removed testing environment --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 284621aa5b1..b3219586188 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -805,7 +805,6 @@ NFS4.x protocol compliance tests: - nfs41_errors=$(( $(echo 0$(sed -n 's/.*testsuite .*errors=\"\([0-9]*\)\".*/+\1/p' xunit-report-v41.xml)) )) - nfs41_failures=$(( $(echo 0$(sed -n 's/.*testsuite .*failures=\"\([0-9]*\)\".*/+\1/p' xunit-report-v41.xml)) )) - exit $(( $nfs40_errors + $nfs41_errors + $nfs40_failures + $nfs41_failures )) - environment: testing artifacts: reports: junit: From 526b6965ebd0ab5e83a9f4be79d52a9d4cc69f49 Mon Sep 17 00:00:00 2001 From: chubnastia Date: Tue, 8 Sep 2026 17:04:12 +0200 Subject: [PATCH 4/6] put testing environment back, where fixed [fast pass] --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b3219586188..fbd1fc539a9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -83,7 +83,7 @@ default: - dcache-dev dependencies: [] rules: - - where: always + - when: always .kubernetes_image: extends: .kubernetes_tags @@ -805,6 +805,7 @@ NFS4.x protocol compliance tests: - nfs41_errors=$(( $(echo 0$(sed -n 's/.*testsuite .*errors=\"\([0-9]*\)\".*/+\1/p' xunit-report-v41.xml)) )) - nfs41_failures=$(( $(echo 0$(sed -n 's/.*testsuite .*failures=\"\([0-9]*\)\".*/+\1/p' xunit-report-v41.xml)) )) - exit $(( $nfs40_errors + $nfs41_errors + $nfs40_failures + $nfs41_failures )) + environment: testing artifacts: reports: junit: From b034d99b3e8a583a81f7138ed752f69428dcb1ac Mon Sep 17 00:00:00 2001 From: chubnastia Date: Tue, 8 Sep 2026 17:22:14 +0200 Subject: [PATCH 5/6] trial with job pick [only: xroot] --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fbd1fc539a9..bf3dccac414 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -97,7 +97,7 @@ default: # -# default cache konfiguration for maven build jobs +# default cache configuration for maven build jobs # Cache downloaded dependencies and plugins between builds. # To keep cache across branches add 'key: "$CI_JOB_NAME"' # From 4fa43fcd0e204673fcfbc9968dddd1fb01f3f5e5 Mon Sep 17 00:00:00 2001 From: chubnastia Date: Tue, 8 Sep 2026 17:41:51 +0200 Subject: [PATCH 6/6] removed needs from Extract coverage job [only: xroot] --- .gitlab-ci.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bf3dccac414..6b6aad766bb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -645,12 +645,6 @@ Extract Coverage: - .fast_pass_rules dependencies: - get_jacoco_cli - needs: - - job: get_jacoco_cli - artifacts: true - optional: true - - job: NFS4.x protocol compliance tests - optional: true script: - JACOCO_CLI_PATH="jacoco-${JACOCO_VERSION}/lib/jacococli.jar" - PODS=$(kubectl -n $K8S_NAMESPACE get pods -l app.kubernetes.io/instance=store -o jsonpath='{.items[*].metadata.name}')