Run the O2DPG simulation tests against a CVMFS release instead of building O2sim

.github/workflows/sim-tests.yml

@@ -0,0 +1,192 @@
+---
+name: Simulation tests
+
+'on':
+  pull_request: {}
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'O2PDPSuite tag to test against (default: newest daily)'
+        type: string
+        required: false
+
+permissions: {}
+
+concurrency:
+  group: sim-tests-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  sim-tests:
+    name: Simulation tests against CVMFS
+    runs-on: [self-hosted, cvmfs]
+    timeout-minutes: 180
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          # The changed-file logic diffs against the merge base, so the full
+          # history is needed, not a shallow clone.
+          fetch-depth: 0
+
+      - name: Resolve the diff base
+        id: base
+        if: github.event_name == 'pull_request'
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          set -eu
+          git rev-parse --verify "$BASE_SHA^{commit}" >/dev/null || {
+            echo "::error title=Cannot resolve diff base::pull request base commit $BASE_SHA does not resolve in this checkout"
+            exit 1
+          }
+          git rev-parse --verify "$HEAD_SHA^{commit}" >/dev/null || {
+            echo "::error title=Cannot resolve diff head::pull request head commit $HEAD_SHA does not resolve in this checkout"
+            exit 1
+          }
+          merge_base=$(git merge-base "$BASE_SHA" "$HEAD_SHA") || {
+            echo "::error title=Cannot compute diff base::git merge-base failed"
+            exit 1
+          }
+          [ -n "$merge_base" ] || {
+            echo "::error title=Cannot compute diff base::merge base is empty"
+            exit 1
+          }
+          echo "sha=$merge_base" >> "$GITHUB_OUTPUT"
+
+      - name: Skip when not relevant or opted into the source build
+        id: gate
+        if: github.event_name == 'pull_request'
+        env:
+          BASE_SHA: ${{ steps.base.outputs.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          set -eu
+          changed=$(git diff --name-only "$BASE_SHA" "$HEAD_SHA")
+          # Opting in means *editing* the existing sentinel, so only count it
+          # when it is Modified. A pull request that Adds it — which is what the
+          # pull request introducing the sentinel does — would otherwise trip its
+          # own opt-out and pass without testing anything.
+          opted=$(git diff --name-only --diff-filter=M "$BASE_SHA" "$HEAD_SHA")
+          if grep -qx 'test/needs-o2-dev' <<< "$opted" ; then
+            echo "::notice title=Skipped::this pull request touches test/needs-o2-dev, so it is tested by build/O2DPG/sim/o2dev against O2 dev instead"
+            echo "skip=true" >> "$GITHUB_OUTPUT"
+          elif ! grep -qE '^(DATA/|MC/|test/|RelVal/)' <<< "$changed" ; then
+            echo "::notice title=Skipped::no changed file matches DATA/, MC/, test/ or RelVal/"
+            echo "skip=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "skip=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Check the CVMFS environment
+        if: steps.gate.outputs.skip != 'true'
+        run: |
+          set -eu
+          test -d /cvmfs/alice.cern.ch || {
+            echo "::error title=CVMFS unavailable::/cvmfs/alice.cern.ch is not mounted on this runner"
+            exit 1
+          }
+          test -x /cvmfs/alice.cern.ch/bin/alienv || {
+            echo "::error title=CVMFS unavailable::/cvmfs/alice.cern.ch/bin/alienv is missing"
+            exit 1
+          }
+
+      - name: Check the AliEn token
+        if: steps.gate.outputs.skip != 'true'
+        env:
+          JALIEN_TOKEN_CERT: /run/alien-ci/tokencert.pem
+          JALIEN_TOKEN_KEY: /run/alien-ci/tokenkey.pem
+        run: |
+          set -eu
+          # DPL's CCDB backend refuses to talk to alice-ccdb.cern.ch without a
+          # token and aborts the whole device, so check up front rather than
+          # letting it surface as a confusing task crash deep in a workflow.
+          for f in "$JALIEN_TOKEN_CERT" "$JALIEN_TOKEN_KEY" ; do
+            test -r "$f" || {
+              echo "::error title=No AliEn token::$f is missing or unreadable. On the runner host: systemctl start alien-ci-token.service"
+              exit 1
+            }
+          done
+          if ! openssl x509 -in "$JALIEN_TOKEN_CERT" -noout -checkend 3600 >/dev/null 2>&1 ; then
+            echo "::error title=AliEn token expiring::the token expires within the hour. On the runner host: systemctl start alien-ci-token.service"
+            exit 1
+          fi
+          echo "AliEn token valid until $(openssl x509 -in "$JALIEN_TOKEN_CERT" -noout -enddate | cut -d= -f2)"
+
+      - name: Resolve the O2PDPSuite tag
+        id: tag
+        if: steps.gate.outputs.skip != 'true'
+        env:
+          REQUESTED_TAG: ${{ inputs.tag }}
+          PR_BODY: ${{ github.event.pull_request.body }}
+        run: |
+          set -eu
+          # shellcheck source=test/ci/resolve_tag.sh
+          . test/ci/resolve_tag.sh
+          moduledir=/cvmfs/alice.cern.ch/el9-x86_64/Modules/modulefiles/O2PDPSuite
+          requested=$REQUESTED_TAG
+          if [ -z "$requested" ]; then
+            # A PR can pin the release with a line "sim-tests-tag: <tag>".
+            requested=$(printf '%s\n' "$PR_BODY" |
+                          sed -n 's/^[[:space:]]*sim-tests-tag:[[:space:]]*//p' | head -n 1 |
+                          tr -d '[:space:]')
+          fi
+          tag=$(resolve_o2pdpsuite_tag "$moduledir" "$requested") || {
+            echo "::error title=No usable O2PDPSuite release::see the message above"
+            exit 1
+          }
+          echo "Testing against O2PDPSuite::$tag"
+          echo "tag=$tag" >> "$GITHUB_OUTPUT"
+
+      - name: Run the O2DPG tests
+        if: steps.gate.outputs.skip != 'true'
+        env:
+          O2PDPSUITE_TAG: ${{ steps.tag.outputs.tag }}
+          O2DPG_TEST_HASH_BASE: ${{ steps.base.outputs.sha }}
+          O2DPG_TEST_HASH_HEAD: ${{ github.event.pull_request.head.sha }}
+          JOBS: 8
+          # o2dpg_sim_workflow.py calls JAlien(['whoami']) purely to fill the
+          # AOD's --created-by field, and only when JALIEN_USER is unset. Setting
+          # it keeps the tests from needing a GRID credential at all, which
+          # matters here: anything readable by this account is readable by the
+          # fork-pull-request code that runs as it.
+          JALIEN_USER: alien-ci
+          # DPL's CCDB backend needs a GRID token for alice-ccdb.cern.ch. These
+          # are *paths* to a short-lived token minted from a service certificate
+          # by a root-owned timer on the runner host; the certificate itself is
+          # never readable by this account. Verified that paths work — the same
+          # variables also accept PEM content, which is what ali-bot passes.
+          JALIEN_TOKEN_CERT: /run/alien-ci/tokencert.pem
+          JALIEN_TOKEN_KEY: /run/alien-ci/tokenkey.pem
+        run: |
+          set -eu
+          # Everything after "-c" is joined into one string and re-evaluated
+          # by the CVMFS alienv via "bash -c \"$*\"". Quoting here is applied
+          # once then discarded, so it is safe for the runner's workspace path,
+          # but a path with a space or "$" would break or double-evaluate.
+          /cvmfs/alice.cern.ch/bin/alienv setenv "O2PDPSuite/$O2PDPSUITE_TAG" -c \
+            env O2DPG_ROOT="$PWD" O2DPG_MC_CONFIG_ROOT="$PWD" \
+                O2DPG_TEST_REPO_DIR="$PWD" \
+                O2DPG_TEST_HASH_BASE="$O2DPG_TEST_HASH_BASE" \
+                O2DPG_TEST_HASH_HEAD="$O2DPG_TEST_HASH_HEAD" \
+                JOBS="$JOBS" \
+                bash test/run_tests.sh
+
+      - name: Upload logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: o2dpg-test-logs
+          path: |
+            o2dpg_tests/**/*.log
+            o2dpg_tests/**/*serverlog*
+            o2dpg_tests/**/*workerlog*
+            o2dpg_tests/**/*mergerlog*
+          if-no-files-found: ignore
+          retention-days: 14
+
+      - name: Prune test artifacts
+        if: always()
+        run: find o2dpg_tests -type f ! -name '*.log' ! -name '*serverlog*' ! -name '*workerlog*' ! -name '*mergerlog*' -delete || true

.github/workflows/syntax-checks.yml

@@ -134,6 +134,24 @@ jobs:
       - name: Run the FileIOGraph test suite
         run: python3 -m unittest discover -s UTILS/FileIOGraph/tests -t UTILS/FileIOGraph/tests
 
+  bash-harnesses:
+    name: Test-harness unit tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run the offline test harnesses
+        run: |
+          error=0
+          for t in test/tests/*.sh ; do
+            echo "::group::$t"
+            bash "$t" || error=1
+            echo "::endgroup::"
+          done
+          exit "$error"
+
   pylint:
     name: Pylint
     runs-on: ubuntu-latest

MC/bin/o2dpg_sim_workflow.py

@@ -5,6 +5,7 @@
 # It aims to handle the different MC possible configurations
 # It just creates a workflow.json txt file, to execute the workflow one must execute right after
 #   ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json
+# The tests covering this script are described in test/README.md.
 #
 # Execution examples:
 #  - pp PYTHIA jets, 2 events, triggered on high pT decay photons on all barrel calorimeters acceptance, eCMS 13 TeV

test/README.md

@@ -7,7 +7,7 @@ At the moment, the tests focus on generator configurations and custom generators
 
 Tests are run via
 ```bash
-${O2DPG_ROOT}/test/run_tests.sh [--fail-immediately]
+${O2DPG_ROOT}/test/run_tests.sh [--fail-immediately] [--keep-artifacts] [SUBTEST...]
 ```
 
 Tests are run for changed
@@ -56,7 +56,32 @@ O2DPG_TEST_REPO_DIR=</path/to/source/O2DPG> ${O2DPG_ROOT}/test/run_tests.sh [--f
 ```
 If you are inside the source directory, you can simply run
 ```bash
-${O2DPG_ROOT}/test/run_tests.sh [--fail-immediately]
+${O2DPG_ROOT}/test/run_tests.sh [--fail-immediately] [--keep-artifacts] [SUBTEST...]
+```
+If the change you are testing is to a test script itself (`run_tests.sh` or
+any `run_*_tests.sh`), invoke the checkout's own entrypoint instead, e.g.
+`bash test/run_tests.sh` from inside the checkout: `run_tests.sh` finds its
+sub-scripts next to itself, so calling `${O2DPG_ROOT}/test/run_tests.sh`
+tests the *released* copy of the script you just edited, not your change,
+even with `O2DPG_TEST_REPO_DIR` pointed at the checkout.
+
+### Running a subset
+
+`run_tests.sh` runs the generator, workflow and RelVal sub-tests. To run only
+some of them, name them:
+
+```bash
+${O2DPG_ROOT}/test/run_tests.sh generator relval
+```
+
+The offline harnesses under `test/tests/` check the entrypoint's selection,
+exit-code aggregation and O2PDPSuite tag resolution without needing an O2
+environment:
+
+```bash
+bash test/tests/run_tests_selection.sh
+bash test/tests/exit_code_aggregation.sh
+bash test/tests/resolve_tag.sh
 ```
 
 ### Keeping all test artifacts
@@ -74,24 +99,33 @@ ${O2DPG_ROOT}/test/run_tests.sh -h
 ```
 which will give you
 ```
-usage: run_tests.sh [--fail-immediately] [--keep-artifacts]
+
+usage: run_tests.sh [--fail-immediately] [--keep-artifacts] [SUBTEST...]
+
+  SUBTEST : one or more of: generator workflow relval (default: all)
 
   FLAGS:
 
-  --fail-immediately : abort as soon as the first tests fails
-  --keep-artifacts : keep simulation and tests artifacts, by default everything but the logs is removed after each test
+  --fail-immediately : stop after the first failing sub-test
+  --keep-artifacts   : keep simulation artifacts, not just the logs
 
   ENVIRONMENT VARIABLES:
 
-  O2DPG_TEST_REPO_DIR : Point to the source repository you want to test.
-  O2DPG_TEST_HASH_BASE : The base hash you want to use for comparison (optional)
-  O2DPG_TEST_HASH_HEAD : The head hash you want to use for comparison (optional)
-
-  If O2DPG_TEST_HASH_BASE is not set, it will be looked for ALIBUILD_BASE_HASH.
-  If also not set, this will be set to HEAD~1. However, if there are unstaged
-  changes, it will be set to HEAD.
+  O2DPG_TEST_REPO_DIR  : the source repository to test
+  O2DPG_TEST_HASH_BASE : base hash for the changed-file diff (optional)
+  O2DPG_TEST_HASH_HEAD : head hash for the changed-file diff (optional)
 
-  If O2DPG_TEST_HASH_HEAD is not set, it will be looked for ALIBUILD_HEAD_HASH.
-  If also not set, this will be set to HEAD. However, if there are unstaged
-  changes, it will left blank.
 ```
+
+## When your change needs an unreleased O2
+
+The `Simulation tests against CVMFS` check runs against a published
+`O2PDPSuite` release, so a change that depends on an unmerged or unreleased O2
+commit cannot pass it. Two escape hatches, in order of preference:
+
+1. If the O2 change is already in a published daily, pin it: add a line
+   `sim-tests-tag: daily-YYYYMMDD-HHMM-1` to the pull request description.
+2. If it is not published anywhere yet, touch `test/needs-o2-dev` with a
+   one-line reason and a link to the O2 pull request. That enables
+   `build/O2DPG/sim/o2dev`, which builds O2 from source against `dev`. It is
+   much slower, so it is opt-in.

test/ci/resolve_tag.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Pick the O2PDPSuite release to test against. Kept free of CVMFS paths and of
+# CI variables so it can be exercised offline.
+
+resolve_o2pdpsuite_tag()
+{
+    local moduledir=${1:-}
+    local requested=${2:-}
+
+    if [[ ! -d "${moduledir}" ]] ; then
+        echo "resolve_o2pdpsuite_tag: no such directory: ${moduledir}" >&2
+        return 1
+    fi
+
+    if [[ -n "${requested}" ]] ; then
+        case ${requested} in
+            *[!A-Za-z0-9._-]* )
+                echo "resolve_o2pdpsuite_tag: invalid tag: ${requested}" >&2
+                return 1 ;;
+        esac
+        if [[ ! -e "${moduledir}/${requested}" ]] ; then
+            echo "resolve_o2pdpsuite_tag: requested tag not available: ${requested}" >&2
+            return 1
+        fi
+        echo "${requested}"
+        return 0
+    fi
+
+    local newest
+    newest=$(find "${moduledir}" -maxdepth 1 -name 'daily-*' -printf '%f\n' 2>/dev/null |
+                 sort -V | tail -n 1)
+    if [[ -z "${newest}" ]] ; then
+        echo "resolve_o2pdpsuite_tag: no daily-* tag in ${moduledir}" >&2
+        return 1
+    fi
+    echo "${newest}"
+}

test/needs-o2-dev

@@ -0,0 +1,5 @@
+Touch this file in a pull request that cannot be tested against a published
+O2PDPSuite release because it depends on an unreleased change in O2.
+Replace this text with a one-line reason and a link to the O2 pull request.
+Touching it enables the build/O2DPG/sim/o2dev check, which builds O2 from
+source against AliceO2Group/AliceO2 dev.