Benchmark prs

.cargo/config.toml

@@ -1,2 +1,5 @@
 [target.'cfg(target_os = "macos")']
 runner = "dev/macos-sign-and-run.sh"
+
+[alias] # command aliases
+ci = ["run", "--quiet", "--package=hyperlight-ci", "--"]

.github/hyperlight-bot.yml

@@ -0,0 +1,8 @@
+# Configuration for the hyperlight-gh-bot GitHub App.
+# See: https://github.com/jprendes/hyperlight-gh-bot
+
+# Name of the artifact containing the comment body.
+artifact_name: "pr-comment"
+
+# Regex matched against the job name to filter which jobs trigger the bot.
+job_filter: "post-benchmark-comment"

.github/workflows/DailyBenchmarks.yml

@@ -12,24 +12,6 @@ permissions:
   actions: read
 
 jobs:
-  # Find the most recent successful run of this workflow so we can download
-  # its benchmark artifacts as a baseline for day-over-day comparison.
-  find-baseline:
-    runs-on: ubuntu-latest
-    outputs:
-      run-id: ${{ steps.find-run.outputs.run_id }}
-    steps:
-      - name: Find latest successful run
-        id: find-run
-        # gh run list returns runs sorted by creation date descending (implicit).
-        # On the first-ever run, this outputs empty and dep_benchmarks.yml
-        # will skip the baseline download (continue-on-error).
-        run: |
-          run_id=$(gh run list --repo "${{ github.repository }}" --workflow DailyBenchmarks.yml --status success --limit 1 --json databaseId --jq '.[0].databaseId // empty')
-          echo "run_id=$run_id" >> "$GITHUB_OUTPUT"
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
   # Build release guest binaries needed by the benchmark suite.
   build-guests:
     uses: ./.github/workflows/dep_build_guests.yml
@@ -38,10 +20,11 @@ jobs:
       arch: X64
       config: release
 
-  # Run benchmarks across all hypervisor/cpu combos, comparing against
-  # the previous day's results. Artifacts are retained for 90 days.
+  # Run benchmarks across all hypervisor/cpu combos. The artifacts they leave
+  # are the baseline for the pull requests that branch from the commits they
+  # measure.
   benchmarks:
-    needs: [build-guests, find-baseline]
+    needs: [build-guests]
     strategy:
       fail-fast: true
       matrix:
@@ -54,8 +37,6 @@ jobs:
       hypervisor: ${{ matrix.hypervisor }}
       cpu_vendor: ${{ matrix.cpu_vendor }}
       arch: ${{ matrix.arch }}
-      baseline_run_id: ${{ needs.find-baseline.outputs.run-id }}
-      retention_days: 90
 
   # File a GitHub issue if any job fails.
   notify-failure:

.github/workflows/ValidatePullRequest.yml

@@ -226,6 +226,70 @@ jobs:
       arch: ${{ matrix.arch }}
       target: ${{ matrix.target }}
 
+  # Run benchmarks and post results as PR comment
+  benchmarks:
+    needs:
+      - docs-pr
+      - build-guests
+    # Required because update-guest-locks is skipped on non-dependabot PRs,
+    # and a skipped dependency transitively skips all downstream jobs.
+    # See: https://github.com/actions/runner/issues/2205
+    if: ${{ !cancelled() && !failure() }}
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [X64]
+        hypervisor: ['hyperv-ws2025', mshv3, kvm]
+        cpu_vendor: [amd, intel]
+    uses: ./.github/workflows/dep_benchmarks.yml
+    secrets: inherit
+    with:
+      docs_only: ${{ needs.docs-pr.outputs.docs-only }}
+      hypervisor: ${{ matrix.hypervisor }}
+      cpu_vendor: ${{ matrix.cpu_vendor }}
+      arch: ${{ matrix.arch }}
+
+  # Report every configuration the run measured against the branch point the
+  # pull request was built from, for the hyperlight-gh-bot to post as a PR
+  # comment. Only runs for PRs (not merge groups) with code changes.
+  benchmark-comment:
+    name: post-benchmark-comment
+    needs:
+      - docs-pr
+      - benchmarks
+    if: ${{ !cancelled() && !failure() && needs.docs-pr.outputs.docs-only == 'false' && github.event_name == 'pull_request' }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      actions: read
+      pull-requests: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+
+      - uses: hyperlight-dev/ci-setup-workflow@2f4142ba17cf573af44fc1e1f1ffc743daded5b3  # v1.10.0
+        with:
+          rust-toolchain: "1.94"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create benchmarks report
+        run: |
+          echo '## Benchmark Results' > pr-comment.md
+          echo '' >> pr-comment.md
+          cargo ci bench-report --config-file bench_report.toml --reproduce \
+            --candidate run:${{ github.run_id }} \
+            --baseline base-of:${{ github.event.pull_request.number }} >> pr-comment.md
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload PR comment artifact
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: pr-comment
+          path: pr-comment.md
+          if-no-files-found: warn
+          retention-days: 1
+
   spelling:
     name: spell check with typos
     runs-on: ubuntu-latest
@@ -254,6 +318,7 @@ jobs:
       - build-test
       - run-examples
       - fuzzing
+      - benchmarks
       - spelling
       - license-headers
     if: always()

.github/workflows/dep_benchmarks.yml

@@ -2,26 +2,12 @@
 
 # Reusable workflow to run benchmarks on a single hypervisor/cpu_vendor combination.
 #
-# Baseline comparison:
-#   The workflow supports two mutually exclusive ways to load a baseline for
-#   Criterion to compare against:
-#
-#   1. baseline_run_id — Downloads benchmark artifacts from a previous workflow
-#      run (by run ID). Used by DailyBenchmarks.yml for day-over-day comparison.
-#
-#   2. baseline_tag — Downloads benchmark tarballs from a GitHub Release (by tag).
-#      If empty (the default), `gh release download` fetches from the latest
-#      stable release. Used by CreateRelease.yml.
-#
-#   If baseline_run_id is set, baseline_tag is ignored.
-#   If neither is set, the latest stable release is used.
-#   Both downloads use continue-on-error so the first-ever run (no baseline
-#   available) succeeds without comparison.
-#
 # Artifact upload:
-#   Benchmark results are always uploaded as workflow artifacts named
-#   benchmarks_<OS>_<hypervisor>_<cpu_vendor>. The retention_days input controls
-#   how long they are kept (default: 5 days).
+#   Benchmark results are uploaded as workflow artifacts named
+#   benchmarks_<OS>_<hypervisor>_<cpu_vendor>, kept for as long as GitHub
+#   allows so that a later run can still be compared against them. Reading
+#   them is left to whoever wants a report, so that a comparison covers every
+#   configuration at once.
 
 name: Run Benchmarks
 
@@ -45,22 +31,6 @@ on:
         description: CPU architecture for the build, X64 or arm64 (passed from caller matrix)
         required: true
         type: string
-      baseline_tag:
-        description: Release tag to download baseline benchmarks from (e.g. dev-latest). Ignored if baseline_run_id is set. If empty, downloads from the latest stable release.
-        required: false
-        type: string
-        default: ""
-      baseline_run_id:
-        description: Workflow run ID to download baseline benchmark artifacts from. Takes precedence over baseline_tag.
-        required: false
-        type: string
-        default: ""
-      retention_days:
-        description: Number of days to retain benchmark artifacts
-        required: false
-        type: number
-        default: 5
-
 env:
   CARGO_TERM_COLOR: always
   RUST_BACKTRACE: full
@@ -123,23 +93,6 @@ jobs:
       - name: Build
         run: just build release
 
-      - name: Download baseline from previous run
-        if: ${{ inputs.baseline_run_id != '' }}
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
-        with:
-          name: benchmarks_${{ runner.os }}_${{ inputs.hypervisor }}_${{ inputs.cpu_vendor }}
-          path: ./target/criterion/
-          run-id: ${{ inputs.baseline_run_id }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-        continue-on-error: true
-
-      - name: Download baseline from release
-        if: ${{ inputs.baseline_run_id == '' }}
-        run: just bench-download ${{ runner.os }} ${{ inputs.hypervisor }} ${{ inputs.cpu_vendor }} ${{ inputs.baseline_tag }}
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        continue-on-error: true
-
       - name: Run benchmarks
         run: just bench-ci main
 
@@ -148,4 +101,6 @@ jobs:
           name: benchmarks_${{ runner.os }}_${{ inputs.hypervisor }}_${{ inputs.cpu_vendor }}
           path: ./target/criterion/
           if-no-files-found: error
-          retention-days: ${{ inputs.retention_days }}
+          # The longest GitHub keeps an artifact, so that a pull request can be
+          # compared against the branch point it was built from.
+          retention-days: 90