diff --git a/.cargo/config.toml b/.cargo/config.toml index b1c8f87b3c..0c3c4d34a0 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,2 +1,5 @@ [target.'cfg(target_os = "macos")'] runner = "dev/macos-sign-and-run.sh" + +[alias] # command aliases +ci = ["run", "--quiet", "--package=hyperlight-ci", "--"] diff --git a/.github/hyperlight-bot.yml b/.github/hyperlight-bot.yml new file mode 100644 index 0000000000..758fc3e550 --- /dev/null +++ b/.github/hyperlight-bot.yml @@ -0,0 +1,8 @@ +# Configuration for the hyperlight-gh-bot GitHub App. +# See: https://github.com/jprendes/hyperlight-gh-bot + +# Name of the artifact containing the comment body. +artifact_name: "pr-comment" + +# Regex matched against the job name to filter which jobs trigger the bot. +job_filter: "post-benchmark-comment" diff --git a/.github/workflows/DailyBenchmarks.yml b/.github/workflows/DailyBenchmarks.yml index 714fe493c0..abf63cfd1b 100644 --- a/.github/workflows/DailyBenchmarks.yml +++ b/.github/workflows/DailyBenchmarks.yml @@ -12,24 +12,6 @@ permissions: actions: read jobs: - # Find the most recent successful run of this workflow so we can download - # its benchmark artifacts as a baseline for day-over-day comparison. - find-baseline: - runs-on: ubuntu-latest - outputs: - run-id: ${{ steps.find-run.outputs.run_id }} - steps: - - name: Find latest successful run - id: find-run - # gh run list returns runs sorted by creation date descending (implicit). - # On the first-ever run, this outputs empty and dep_benchmarks.yml - # will skip the baseline download (continue-on-error). - run: | - run_id=$(gh run list --repo "${{ github.repository }}" --workflow DailyBenchmarks.yml --status success --limit 1 --json databaseId --jq '.[0].databaseId // empty') - echo "run_id=$run_id" >> "$GITHUB_OUTPUT" - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # Build release guest binaries needed by the benchmark suite. build-guests: uses: ./.github/workflows/dep_build_guests.yml @@ -38,10 +20,11 @@ jobs: arch: X64 config: release - # Run benchmarks across all hypervisor/cpu combos, comparing against - # the previous day's results. Artifacts are retained for 90 days. + # Run benchmarks across all hypervisor/cpu combos. The artifacts they leave + # are the baseline for the pull requests that branch from the commits they + # measure. benchmarks: - needs: [build-guests, find-baseline] + needs: [build-guests] strategy: fail-fast: true matrix: @@ -54,8 +37,6 @@ jobs: hypervisor: ${{ matrix.hypervisor }} cpu_vendor: ${{ matrix.cpu_vendor }} arch: ${{ matrix.arch }} - baseline_run_id: ${{ needs.find-baseline.outputs.run-id }} - retention_days: 90 # File a GitHub issue if any job fails. notify-failure: diff --git a/.github/workflows/ValidatePullRequest.yml b/.github/workflows/ValidatePullRequest.yml index 0c8d75a2dc..571b5683f1 100644 --- a/.github/workflows/ValidatePullRequest.yml +++ b/.github/workflows/ValidatePullRequest.yml @@ -226,6 +226,70 @@ jobs: arch: ${{ matrix.arch }} target: ${{ matrix.target }} + # Run benchmarks and post results as PR comment + benchmarks: + needs: + - docs-pr + - build-guests + # Required because update-guest-locks is skipped on non-dependabot PRs, + # and a skipped dependency transitively skips all downstream jobs. + # See: https://github.com/actions/runner/issues/2205 + if: ${{ !cancelled() && !failure() }} + strategy: + fail-fast: false + matrix: + arch: [X64] + hypervisor: ['hyperv-ws2025', mshv3, kvm] + cpu_vendor: [amd, intel] + uses: ./.github/workflows/dep_benchmarks.yml + secrets: inherit + with: + docs_only: ${{ needs.docs-pr.outputs.docs-only }} + hypervisor: ${{ matrix.hypervisor }} + cpu_vendor: ${{ matrix.cpu_vendor }} + arch: ${{ matrix.arch }} + + # Report every configuration the run measured against the branch point the + # pull request was built from, for the hyperlight-gh-bot to post as a PR + # comment. Only runs for PRs (not merge groups) with code changes. + benchmark-comment: + name: post-benchmark-comment + needs: + - docs-pr + - benchmarks + if: ${{ !cancelled() && !failure() && needs.docs-pr.outputs.docs-only == 'false' && github.event_name == 'pull_request' }} + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + pull-requests: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - uses: hyperlight-dev/ci-setup-workflow@2f4142ba17cf573af44fc1e1f1ffc743daded5b3 # v1.10.0 + with: + rust-toolchain: "1.94" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Create benchmarks report + run: | + echo '## Benchmark Results' > pr-comment.md + echo '' >> pr-comment.md + cargo ci bench-report --config-file bench_report.toml --reproduce \ + --candidate run:${{ github.run_id }} \ + --baseline base-of:${{ github.event.pull_request.number }} >> pr-comment.md + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload PR comment artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: pr-comment + path: pr-comment.md + if-no-files-found: warn + retention-days: 1 + spelling: name: spell check with typos runs-on: ubuntu-latest @@ -254,6 +318,7 @@ jobs: - build-test - run-examples - fuzzing + - benchmarks - spelling - license-headers if: always() diff --git a/.github/workflows/dep_benchmarks.yml b/.github/workflows/dep_benchmarks.yml index dc81ed9705..5e54ecbb35 100644 --- a/.github/workflows/dep_benchmarks.yml +++ b/.github/workflows/dep_benchmarks.yml @@ -2,26 +2,12 @@ # Reusable workflow to run benchmarks on a single hypervisor/cpu_vendor combination. # -# Baseline comparison: -# The workflow supports two mutually exclusive ways to load a baseline for -# Criterion to compare against: -# -# 1. baseline_run_id — Downloads benchmark artifacts from a previous workflow -# run (by run ID). Used by DailyBenchmarks.yml for day-over-day comparison. -# -# 2. baseline_tag — Downloads benchmark tarballs from a GitHub Release (by tag). -# If empty (the default), `gh release download` fetches from the latest -# stable release. Used by CreateRelease.yml. -# -# If baseline_run_id is set, baseline_tag is ignored. -# If neither is set, the latest stable release is used. -# Both downloads use continue-on-error so the first-ever run (no baseline -# available) succeeds without comparison. -# # Artifact upload: -# Benchmark results are always uploaded as workflow artifacts named -# benchmarks___. The retention_days input controls -# how long they are kept (default: 5 days). +# Benchmark results are uploaded as workflow artifacts named +# benchmarks___, kept for as long as GitHub +# allows so that a later run can still be compared against them. Reading +# them is left to whoever wants a report, so that a comparison covers every +# configuration at once. name: Run Benchmarks @@ -45,22 +31,6 @@ on: description: CPU architecture for the build, X64 or arm64 (passed from caller matrix) required: true type: string - baseline_tag: - description: Release tag to download baseline benchmarks from (e.g. dev-latest). Ignored if baseline_run_id is set. If empty, downloads from the latest stable release. - required: false - type: string - default: "" - baseline_run_id: - description: Workflow run ID to download baseline benchmark artifacts from. Takes precedence over baseline_tag. - required: false - type: string - default: "" - retention_days: - description: Number of days to retain benchmark artifacts - required: false - type: number - default: 5 - env: CARGO_TERM_COLOR: always RUST_BACKTRACE: full @@ -123,23 +93,6 @@ jobs: - name: Build run: just build release - - name: Download baseline from previous run - if: ${{ inputs.baseline_run_id != '' }} - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: benchmarks_${{ runner.os }}_${{ inputs.hypervisor }}_${{ inputs.cpu_vendor }} - path: ./target/criterion/ - run-id: ${{ inputs.baseline_run_id }} - github-token: ${{ secrets.GITHUB_TOKEN }} - continue-on-error: true - - - name: Download baseline from release - if: ${{ inputs.baseline_run_id == '' }} - run: just bench-download ${{ runner.os }} ${{ inputs.hypervisor }} ${{ inputs.cpu_vendor }} ${{ inputs.baseline_tag }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - continue-on-error: true - - name: Run benchmarks run: just bench-ci main @@ -148,4 +101,6 @@ jobs: name: benchmarks_${{ runner.os }}_${{ inputs.hypervisor }}_${{ inputs.cpu_vendor }} path: ./target/criterion/ if-no-files-found: error - retention-days: ${{ inputs.retention_days }} + # The longest GitHub keeps an artifact, so that a pull request can be + # compared against the branch point it was built from. + retention-days: 90 diff --git a/Cargo.lock b/Cargo.lock index de32f7fe70..1299050f54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -57,6 +57,16 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +[[package]] +name = "ansi-replace" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8b155ab93213f41c886d3a46e335258428e52c7cf868e25cf099d50274496d" +dependencies = [ + "regex", + "stable-pattern", +] + [[package]] name = "anstream" version = "1.0.0" @@ -93,7 +103,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -104,7 +114,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -498,6 +508,19 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "console" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width", + "windows-sys 0.59.0", +] + [[package]] name = "const-oid" version = "0.10.2" @@ -592,6 +615,19 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "cpu-pin" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb5bc1be026f7f066429ce0611e23a341db91b91ed701de4a1432d01d3ed1105" +dependencies = [ + "libc", + "mach2", + "once_cell", + "tokio", + "windows", +] + [[package]] name = "cpufeatures" version = "0.3.1" @@ -635,6 +671,17 @@ dependencies = [ "walkdir", ] +[[package]] +name = "criterion-markdown" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c723767825fca388c4d768dfbe983d7ba3f7e0b3104292f85c8ca8bd6880822" +dependencies = [ + "anyhow", + "serde", + "serde_json", +] + [[package]] name = "criterion-plot" version = "0.8.2" @@ -645,6 +692,23 @@ dependencies = [ "itertools 0.13.0", ] +[[package]] +name = "criterion-swarm" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bbb01e7dd0d588f78fe0bd2fdc12e921875542754473f22b09da4f9edbc2770" +dependencies = [ + "ansi-replace", + "anyhow", + "clap", + "cpu-pin", + "indicatif", + "regex", + "serde_json", + "simple-pool", + "tokio", +] + [[package]] name = "crossbeam-channel" version = "0.5.17" @@ -840,7 +904,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -886,6 +950,12 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "endian-type" version = "0.1.2" @@ -928,7 +998,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1275,7 +1345,7 @@ dependencies = [ "gobject-sys", "libc", "system-deps", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1546,6 +1616,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "hyperlight-ci" +version = "0.0.0" +dependencies = [ + "anyhow", + "clap", + "criterion-markdown", + "criterion-swarm", + "regex", + "serde", + "serde_json", + "tokio", + "toml 1.1.6+spec-1.1.0", +] + [[package]] name = "hyperlight-common" version = "0.17.0" @@ -1731,7 +1816,7 @@ dependencies = [ "vmm-sys-util", "windows", "windows-result", - "windows-sys", + "windows-sys 0.61.2", "windows-version", ] @@ -1933,6 +2018,19 @@ dependencies = [ "serde_core", ] +[[package]] +name = "indicatif" +version = "0.17.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" +dependencies = [ + "console", + "number_prefix", + "portable-atomic", + "unicode-width", + "web-time", +] + [[package]] name = "ipnet" version = "2.12.2" @@ -2391,7 +2489,7 @@ checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" dependencies = [ "libc", "wasi", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -2455,7 +2553,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -2488,6 +2586,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "object" version = "0.40.0" @@ -2499,6 +2603,12 @@ dependencies = [ "ruzstd", ] +[[package]] +name = "object-id" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c587bd1cd63959a8520442afc0f92a875d83deea175c7b48dd9f104a2c5070a9" + [[package]] name = "oci-spec" version = "0.10.0" @@ -3347,7 +3457,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -3558,6 +3668,16 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" +[[package]] +name = "simple-pool" +version = "0.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "073382259dbeb56c3eaab04a1d330459f6490d1e518b2a8ee441c8bd00dbc092" +dependencies = [ + "object-id", + "parking_lot", +] + [[package]] name = "sketches-ddsketch" version = "0.3.1" @@ -3583,7 +3703,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -3601,6 +3721,15 @@ dependencies = [ "lock_api", ] +[[package]] +name = "stable-pattern" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4564168c00635f88eaed410d5efa8131afa8d8699a612c80c455a0ba05c21045" +dependencies = [ + "memchr", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -3702,7 +3831,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -3777,7 +3906,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -4459,7 +4588,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -4569,6 +4698,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -4578,6 +4716,22 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + [[package]] name = "windows-threading" version = "0.2.1" @@ -4596,6 +4750,54 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "winnow" version = "0.7.15" diff --git a/Cargo.toml b/Cargo.toml index 2660825a04..0b1c83201f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ default-members = [ "src/hyperlight_testing", ] members = [ + "src/hyperlight_ci", "src/hyperlight_common", "src/hyperlight_guest", "src/hyperlight_host", diff --git a/Justfile b/Justfile index 577e24b8b9..5859933302 100644 --- a/Justfile +++ b/Justfile @@ -423,26 +423,12 @@ tar-static-lib: (build-rust-capi "release") (build-rust-capi "debug") ### BENCHMARKING ### #################### -# Warning: can overwrite previous local benchmarks, so run this before running benchmarks -# Downloads the benchmarks result from the given release tag. -# If tag is not given, defaults to latest release -# Options for os: "Windows", or "Linux" -# Options for Linux hypervisor: "kvm", "mshv3" -# Options for Windows hypervisor: "hyperv", "hyperv-ws2025" -# Options for cpu_vendor: "amd", "intel" -bench-download os hypervisor cpu_vendor tag="": - gh release download {{ tag }} -D ./target/ -p benchmarks_{{ os }}_{{ hypervisor }}_{{ cpu_vendor }}.tar.gz - mkdir -p target/criterion {{ if os() == "windows" { "-Force" } else { "" } }} - tar -zxvf target/benchmarks_{{ os }}_{{ hypervisor }}_{{ cpu_vendor }}.tar.gz -C target/criterion/ --strip-components=1 - # Warning: compares to and then OVERWRITES the given baseline bench-ci baseline features="": - @# Benchmarks are always run with release builds for meaningful results - cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose --save-baseline {{ baseline }} + cargo ci bench {{ if features == "" {''} else { "--features " + features } }} --verbose --save-baseline {{ baseline }} bench features="": - @# Benchmarks are always run with release builds for meaningful results - cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose + cargo ci bench {{ if features == "" {''} else { "--features " + features } }} --verbose ############### ### FUZZING ### diff --git a/bench_report.toml b/bench_report.toml new file mode 100644 index 0000000000..de30b510b5 --- /dev/null +++ b/bench_report.toml @@ -0,0 +1,46 @@ +# Benchmarks whose results appear in pull request comments, as regular +# expressions matched against criterion ids. A benchmark is reported when it +# matches `allowlist` and no `denylist` entry, and an allowlist entry matching +# nothing fails the report. +allowlist = [ + "^function_call_codec/decode_vec_bytes_copy$", + "^function_call_codec/encode_control/byte_chunks$", + "^function_call_codec/encode_control/vec_bytes$", + "^payload_allocation/slot_pool_segmented/262144$", + "^payload_allocation/slot_pool_segmented/65536$", + "^sandboxes/create_initialized_and_drop/medium$", + "^slot_pool/alloc_dealloc_128$", + "^slot_pool/alloc_dealloc_1500$", + "^slot_pool/alloc_dealloc_4096$", + "^snapshot_files/load_snapshot_unverified/small$", + "^virtq_readonly/slot_pool_segmented/262144$", + "^virtq_readonly/slot_pool_segmented/65536$", + "^virtq_readonly/slot_pool_segmented_fragmented/262144$", + "^virtq_readonly/slot_pool_segmented_fragmented/65536$", + "^virtq_readwrite/slot_pool_segmented/262144$", + "^virtq_readwrite/slot_pool_segmented/65536$", + "^virtq_readwrite/slot_pool_segmented/8192$", + "^virtq_readwrite/slot_pool_segmented_fragmented/65536$", + "^virtq_readwrite/slot_pool_segmented_fragmented/8192$", +] + +# Every entry names one benchmark, so nothing needs excluding. +denylist = [] + +# How far a result moves before a report calls it a change rather than noise. +# The improvements are how many times faster, `regression` the fraction of the +# baseline a result falls to. +improvement = 1.5 # 1.5x faster +strong_improvement = 1.8 # 1.8x faster +regression = 0.67 # 1.5x slower + +# Whose runs `run:`, `pr:` and the rest read, `/` or `remote:` +# for whichever repository a git remote points at. +repo = "hyperlight-dev/hyperlight" + +# How many changes are called out before the tables, none at 0. +summary_limit = 3 + +# End a report with the command that asks for it again, as CI does for the +# pull request comment. +reproduce = false diff --git a/docs/benchmarking-hyperlight.md b/docs/benchmarking-hyperlight.md index 811ef18f06..dfa8c3012e 100644 --- a/docs/benchmarking-hyperlight.md +++ b/docs/benchmarking-hyperlight.md @@ -5,7 +5,7 @@ Hyperlight uses the [Criterion](https://bheisler.github.io/criterion.rs/book/ind ## When Benchmarks are run 1. Daily (scheduled) - - Benchmarks run daily via `DailyBenchmarks.yml`, comparing results against the previous day's run. Results are stored as workflow artifacts with 90-day retention. + - Benchmarks run daily via `DailyBenchmarks.yml`. Results are stored as workflow artifacts with 90-day retention, and are what pull requests are compared against. ``` sandboxes/create_sandbox @@ -14,7 +14,10 @@ Hyperlight uses the [Criterion](https://bheisler.github.io/criterion.rs/book/ind Change within noise threshold.* ``` -2. For each release +2. For each pull request + - Benchmarks run on every hypervisor and cpu vendor in `ValidatePullRequest.yml`, which invokes `dep_benchmarks.yml`. The results are reported against the daily benchmarks of the commit the pull request branched from, and posted as a comment. + +3. For each release - For each release, benchmarks are run as part of the release pipeline in `CreateRelease.yml`, which invokes `dep_benchmarks.yml`. These benchmark results are compared to the previous release, and are uploaded as part of the "Release assets" on the GitHub release page. Currently, benchmarks are run on windows, linux-kvm (ubuntu), and linux-hyperv (mariner). Only release builds are benchmarked, not debug. @@ -72,6 +75,19 @@ Found 1 outliers among 100 measurements (1.00%) ## Running benchmarks locally -Use `just bench` to run benchmarks with release builds (the only supported configuration). Comparing local benchmark results to GitHub-saved benchmarks doesn't make much sense, since you'd be using different hardware, but you can use `just bench-download os hypervisor cpu_vendor [tag] ` to download and extract the GitHub release benchmarks to the correct folder. You can then run `just bench-ci main` to compare to (and overwrite) the previous release benchmarks. Note that `main` is the name of the baselines stored in GitHub. +Use `just bench` to run benchmarks with release builds (the only supported configuration). Comparing local benchmark results to the ones CI measures doesn't say much, since you'd be using different hardware, but `cargo ci bench-report` fetches them for you. + +`cargo ci bench-report` renders the comparison. `--candidate` and `--baseline` say where each side comes from: a criterion directory, `run:` for a CI run, `pr:` for the latest run of a pull request, `commit:` for the benchmarks of the default branch taken at or before a commit, `base-of:` for the ones taken where a pull request branched, or `release:` for the ones a release carries. Criterion keeps the last run of a directory in `new` and the one before it in `base`, so a directory on its own reports the last run against the previous one. + +The default branch is benchmarked daily rather than per commit, so `commit:` and `base-of:` take the closest run that does not carry changes the commit never had. A pull request defaults to the branch point it was built from, since nothing within its own results says what they mean. Workflow artifacts are swept away after 90 days, so reaching further back means the results a release carries. CI results cover every hypervisor and cpu vendor, so results are paired with the ones measured on the same kind of machine. A run records its own operating system, cpu vendor and hypervisor, and CI artifacts are named after the configuration that produced them. Results that fit no counterpart, or several, are reported without a comparison. + +```sh +# a pull request against the branch point it was built from +cargo ci bench-report --candidate pr:1529 +# this machine against the configuration in CI that matches it +cargo ci bench-report --baseline pr:1529 +# this machine against what a release measured +cargo ci bench-report --baseline release:v0.17.0 +``` **Important**: The `just bench` command uses release builds by default to ensure meaningful performance measurements. For profiling purposes, you can compile benchmarks with debug symbols by running `cargo bench` directly. diff --git a/src/hyperlight_ci/Cargo.toml b/src/hyperlight_ci/Cargo.toml new file mode 100644 index 0000000000..087bcdd5f8 --- /dev/null +++ b/src/hyperlight_ci/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "hyperlight-ci" +edition = "2021" +# fields intentionally not set, to avoid accidentally publishing this crate to crates.io +description = """ +Hyperlight's CI and development tools. +""" + +[lints] +workspace = true + +[dependencies] +anyhow = "1" +clap = { version = "4.6.1", features = ["derive"] } +criterion-markdown = "0.2.1" +criterion-swarm = "0.2.2" +regex = "1" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +tokio = { version = "1", features = ["rt", "macros"] } +toml = "1" \ No newline at end of file diff --git a/src/hyperlight_ci/src/ballast.rs b/src/hyperlight_ci/src/ballast.rs new file mode 100644 index 0000000000..32113fca7c --- /dev/null +++ b/src/hyperlight_ci/src/ballast.rs @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2025 The Hyperlight Authors. +//! A resident VM held for the duration of a benchmark run. + +use std::io::{BufRead, BufReader}; +use std::path::PathBuf; +use std::process::{Child, ChildStdin, Command, Stdio}; + +use anyhow::{Context, bail}; + +/// Line the helper prints once its sandbox is live. +const READY: &str = "ballast ready"; + +/// A sandbox held alive for a whole benchmark run. +/// +/// Benchmarks that create and drop sandboxes leave the host with no resident +/// VM between iterations. Crossing that boundary toggles a kernel static key, +/// and each toggle patches kernel text and IPIs every core. That cost lands on +/// whichever benchmark happens to trigger it, so holding one sandbox resident +/// keeps it out of the measurements. +pub(crate) struct Ballast { + child: Child, + /// The helper exits when this closes, which the OS does for us if this + /// process is killed before [`Drop`] can run. + _stdin: ChildStdin, +} + +impl Ballast { + /// Build the helper, start it, and wait until its sandbox is live. + pub(crate) fn start() -> anyhow::Result { + let exe = build().context("Failed to build the ballast helper")?; + + let mut child = Command::new(&exe) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .with_context(|| format!("Failed to start the ballast helper {}", exe.display()))?; + + let stdin = child.stdin.take().expect("stdin is piped"); + let stdout = child.stdout.take().expect("stdout is piped"); + let mut line = String::new(); + BufReader::new(stdout) + .read_line(&mut line) + .context("Failed to read readiness from the ballast helper")?; + + if line.trim() != READY { + bail!("Ballast helper did not become ready"); + } + + Ok(Self { + child, + _stdin: stdin, + }) + } +} + +impl Drop for Ballast { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +/// Build the ballast example and return the executable cargo produced. +fn build() -> anyhow::Result { + let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".to_string()); + + let output = Command::new(cargo) + .args([ + "build", + "--release", + "--package", + "hyperlight-host", + "--example", + "ballast", + "--message-format=json-render-diagnostics", + ]) + .stderr(Stdio::inherit()) + .output()?; + + if !output.status.success() { + bail!("cargo build failed for the ballast helper"); + } + + for line in output.stdout.as_slice().lines() { + let Ok(line) = line else { continue }; + let Ok(msg) = serde_json::from_str::(&line) else { + continue; + }; + if msg["reason"] == "compiler-artifact" && msg["target"]["name"] == "ballast" { + if let Some(exe) = msg["executable"].as_str() { + return Ok(PathBuf::from(exe)); + } + } + } + + bail!("cargo reported no executable for the ballast helper") +} diff --git a/src/hyperlight_ci/src/bench.rs b/src/hyperlight_ci/src/bench.rs new file mode 100644 index 0000000000..6fe4c99390 --- /dev/null +++ b/src/hyperlight_ci/src/bench.rs @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2025 The Hyperlight Authors. +//! The `bench` subcommand: runs criterion benchmarks in parallel via criterion-swarm. + +use std::collections::HashSet; +use std::io::IsTerminal; +use std::path::PathBuf; + +use anyhow::Context; +use criterion_swarm::{CriterionSwarm, OutputMode}; + +use crate::ballast::Ballast; +use crate::config::BenchConfig; +use crate::manifest; + +/// An output mode flag for `--build-output` / `--benchmarks-output`. +#[derive(Clone, Debug)] +pub(crate) struct OutputModeFlags(OutputMode); + +impl OutputModeFlags { + /// Parse a single token into an `OutputMode` flag. + fn parse_one(s: &str) -> Result { + match s.trim().to_ascii_lowercase().as_str() { + "spinner" => Ok(OutputMode::SPINNER), + "stream" => Ok(OutputMode::STREAM), + "summary" => Ok(OutputMode::SUMMARY), + "none" | "silent" => Ok(OutputMode::SILENT), + other => Err(format!( + "unknown output mode `{other}` (expected: spinner, stream, summary, none, silent)" + )), + } + } +} + +impl std::str::FromStr for OutputModeFlags { + type Err = String; + fn from_str(s: &str) -> Result { + let mut mode = OutputMode::SILENT; + for part in s.split(',') { + mode |= Self::parse_one(part)?; + } + Ok(Self(mode)) + } +} + +/// Merge a `Vec` into a single `OutputMode` by OR-ing them together. +fn merge_output_modes(flags: &[OutputModeFlags]) -> OutputMode { + flags.iter().fold(OutputMode::SILENT, |acc, f| acc | f.0) +} + +/// Command-line arguments for the `bench` subcommand. +#[derive(clap::Args)] +pub struct BenchArgs { + /// Pre-built benchmark binary to use (skip build step; can be specified multiple times) + #[arg(long)] + pub binary: Vec, + + /// Number of benchmarks to run in parallel (0 = all P-cores, default: 0) + #[arg(long, short, default_value_t = 0)] + pub jobs: usize, + + /// Build output mode (comma-separated or repeated): spinner, stream, summary, none + #[arg(long, value_delimiter = ',')] + pub build_output: Vec, + + /// Benchmarks output mode (comma-separated or repeated): spinner, stream, summary, none + #[arg(long, value_delimiter = ',')] + pub benchmarks_output: Vec, + + /// Additional features to pass to cargo when building benchmarks (can be specified multiple times) + #[arg(short = 'F', long)] + pub features: Vec, + + /// Run only the benchmarks selected by this config file + #[arg(long, value_name = "PATH")] + pub config_file: Option, + + /// Run without holding a sandbox resident for the duration of the run + #[arg(long)] + pub no_ballast: bool, + + /// Additional arguments to forward to criterion benchmarks + #[arg(trailing_var_arg = true, allow_hyphen_values = true)] + pub bench_args: Vec, +} + +pub async fn run(mut args: BenchArgs) -> anyhow::Result<()> { + let config = args + .config_file + .as_deref() + .map(BenchConfig::load) + .transpose()?; + + let mut swarm = CriterionSwarm::builder().jobs(args.jobs); + + if !args.binary.is_empty() { + swarm = swarm.binaries(args.binary); + } + + if !args.features.is_empty() { + swarm = swarm.build_args(["--features".to_string(), args.features.join(",")]); + } + + for arg in args.bench_args { + swarm = swarm.bench_arg(arg); + } + + if args.build_output.is_empty() { + let mode = if std::io::stderr().is_terminal() { + OutputMode::SPINNER | OutputMode::SUMMARY + } else { + OutputMode::STREAM | OutputMode::SUMMARY + }; + args.build_output.push(OutputModeFlags(mode)); + } + + if args.benchmarks_output.is_empty() { + let mode = if std::io::stderr().is_terminal() { + OutputMode::SPINNER | OutputMode::STREAM | OutputMode::SUMMARY + } else { + OutputMode::STREAM | OutputMode::SUMMARY + }; + args.benchmarks_output.push(OutputModeFlags(mode)); + } + + let build_mode = merge_output_modes(&args.build_output); + let bench_mode = merge_output_modes(&args.benchmarks_output); + swarm = swarm.output( + criterion_swarm::ProgressReporter::new() + .build(build_mode) + .benchmarks(bench_mode), + ); + + let mut swarm = swarm + .prepare() + .await + .context("Failed to prepare criterion swarm")?; + + if let Some(config) = &config { + let selected: HashSet = config + .select(swarm.benchmarks().into_iter().map(str::to_string))? + .into_iter() + .collect(); + swarm.retain(|name| selected.contains(name)); + } + + if bench_mode == (bench_mode | OutputMode::SUMMARY) { + let total = swarm.benchmarks().len(); + let jobs = swarm.jobs().min(total); + println!("Running {total} benchmarks with parallelism {jobs}"); + } + + manifest::write(swarm.benchmarks().into_iter().map(str::to_string)) + .context("Failed to write the benchmark manifest")?; + + // Held until the run finishes. + let ballast = if args.no_ballast { + None + } else { + Some(Ballast::start()?) + }; + + let result = swarm.run().await.context("Failed to run criterion swarm"); + drop(ballast); + result +} diff --git a/src/hyperlight_ci/src/bench_report.rs b/src/hyperlight_ci/src/bench_report.rs new file mode 100644 index 0000000000..653fe709a6 --- /dev/null +++ b/src/hyperlight_ci/src/bench_report.rs @@ -0,0 +1,577 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2025 The Hyperlight Authors. +//! The `bench-report` subcommand: generates a markdown table from existing +//! criterion benchmark results in `target/criterion/`. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::str::FromStr; + +use anyhow::{Context, Result}; +use clap::Args; +use criterion_swarm::{CriterionSwarm, NoopReporter}; + +use crate::config::BenchConfig; +use crate::{manifest, remote}; + +/// Where downloaded runs are kept. +const RUN_CACHE: &str = "target/ci-runs"; + +/// Whose runs a report reads, when nothing else says. +const DEFAULT_REPO: &str = "hyperlight-dev/hyperlight"; + +/// Where results come from, either a criterion directory or CI. +#[derive(Clone)] +pub enum Source { + Dir(PathBuf), + Run(u64), + PullRequest(u64), + /// Benchmarks of the default branch taken at or before a commit. + Commit(String), + /// Benchmarks of the default branch taken where a pull request branched. + BaseOf(u64), + /// Benchmarks a release carries, which outlive the workflow artifacts. + Release(String), +} + +impl FromStr for Source { + type Err = String; + + fn from_str(value: &str) -> Result { + // Anything else is a path, so windows drive letters stay paths. + let Some((kind @ ("run" | "pr" | "commit" | "base-of" | "release"), rest)) = + value.split_once(':') + else { + return Ok(Self::Dir(value.into())); + }; + + match kind { + "commit" => return Ok(Self::Commit(rest.to_string())), + "release" => return Ok(Self::Release(rest.to_string())), + _ => {} + } + + let id = rest + .parse() + .map_err(|_| format!("`{rest}` is not a {kind} number"))?; + Ok(match kind { + "run" => Self::Run(id), + "pr" => Self::PullRequest(id), + _ => Self::BaseOf(id), + }) + } +} + +/// Results to report, identified by the host that produced them. +struct Input { + label: Option, + dir: PathBuf, + host: Option, +} + +/// A set of results and the commit they were taken at. +struct Origin { + commit: Option, + /// How to ask for these same results again, whatever was asked for here. + pinned: String, + inputs: Vec, +} + +/// What distinguishes one set of benchmark results from another. +#[derive(PartialEq)] +struct Identity { + os: String, + /// `amd` or `intel`. + vendor: String, + hypervisor: Option, +} + +impl Identity { + /// Read from the name CI gives an artifact, `Linux_kvm_amd` and so on. + fn from_label(label: &str) -> Option { + let (os, rest) = label.split_once('_')?; + let (hypervisor, vendor) = rest.rsplit_once('_')?; + let os = os.to_lowercase(); + Some(Self { + hypervisor: Some(canonical_hypervisor(&os, hypervisor)), + os, + vendor: vendor.to_lowercase(), + }) + } + + /// Read from what a run recorded about the machine it ran on. + fn from_host(host: &manifest::Host) -> Option { + let vendor = match host.cpu_vendor.as_deref()? { + vendor if vendor.contains("AMD") => "amd", + vendor if vendor.contains("Intel") => "intel", + _ => return None, + }; + Some(Self { + hypervisor: host + .hypervisor + .as_deref() + .map(|name| canonical_hypervisor(&host.os, name)), + os: host.os.clone(), + vendor: vendor.to_string(), + }) + } + + /// Whether both could be the same machine. What one of them does not say + /// cannot contradict the other. + fn matches(&self, other: &Self) -> bool { + self.os == other.os + && self.vendor == other.vendor + && match (&self.hypervisor, &other.hypervisor) { + (Some(ours), Some(theirs)) => ours == theirs, + _ => true, + } + } +} + +/// Windows runs on whp alone, so its artifacts are named after the runner +/// image instead. Elsewhere the name carries a version, `mshv3` for `mshv`. +fn canonical_hypervisor(os: &str, name: &str) -> String { + match os { + "windows" => "whp".to_string(), + _ => name.trim_end_matches(char::is_numeric).to_string(), + } +} + +/// Command-line arguments for the `bench-report` subcommand. +#[derive(Args)] +pub struct BenchReportArgs { + /// Benchmark binary to list benchmarks from (can be specified multiple times). + /// When provided, only benchmarks available in these binaries are included. + #[arg(long)] + pub binary: Vec, + + /// Results to report: a criterion directory, `run:`, `pr:`, + /// `commit:`, `base-of:` or `release:` + #[arg(long, value_name = "SOURCE", default_value = "target/criterion")] + pub candidate: Source, + + /// Results to compare against, in the same forms as the candidate. Defaults + /// to where a pull request branched, and otherwise to the previous run held + /// in the reported directory. + #[arg(long, value_name = "SOURCE")] + pub baseline: Option, + + /// Repository holding the CI runs, `/` or `remote:` for + /// whichever one a git remote points at [default: hyperlight-dev/hyperlight] + #[arg(long, value_name = "REPO")] + pub repo: Option, + + /// Wrap the output in a collapsible
tag with the given summary text. + #[arg(long)] + pub collapsible: Option, + + /// Report only the benchmarks selected by this config file + #[arg(long, value_name = "PATH")] + pub config_file: Option, + + /// Call a result improved once it is this many times faster [default: 1.1] + #[arg(long, value_name = "RATIO")] + pub improvement: Option, + + /// Call an improvement strong once it is this many times faster [default: 1.8] + #[arg(long, value_name = "RATIO")] + pub strong_improvement: Option, + + /// Call a result regressed once it is this fraction of the baseline [default: 0.9] + #[arg(long, value_name = "RATIO")] + pub regression: Option, + + /// How many changes to call out before the tables, none at 0 [default: 3] + #[arg(long, value_name = "COUNT")] + pub summary_limit: Option, + + /// End the report with the command that asks for it again + #[arg(long, value_name = "BOOL", num_args = 0..=1, default_missing_value = "true")] + pub reproduce: Option, + + /// Additional arguments to forward to criterion benchmarks (e.g. filter, --exact) + #[arg(trailing_var_arg = true, allow_hyphen_values = true)] + pub bench_args: Vec, +} + +/// Entry point for the bench-report subcommand. +pub async fn run(args: BenchReportArgs) -> Result<()> { + let config = args + .config_file + .as_deref() + .map(BenchConfig::load) + .transpose()?; + let thresholds = thresholds(&args, config.as_ref()); + let repo = args + .repo + .clone() + .or_else(|| config.as_ref().and_then(|c| c.repo.clone())) + .unwrap_or_else(|| DEFAULT_REPO.to_string()); + let repo = remote::repository(&repo)?; + let summary_limit = args + .summary_limit + .or_else(|| config.as_ref().and_then(|c| c.summary_limit)); + let reproduce_wanted = args + .reproduce + .or_else(|| config.as_ref().and_then(|c| c.reproduce)) + .unwrap_or(false); + + let candidate = resolve(&args.candidate, &repo)?; + + // Nothing within a pull request's results says what they mean, so they are + // measured against the branch point they were built from. + let source = args.baseline.clone().or(match &args.candidate { + Source::PullRequest(pull_request) => Some(Source::BaseOf(*pull_request)), + _ => None, + }); + + let mut baseline = Origin { + commit: None, + pinned: String::new(), + inputs: Vec::new(), + }; + if let Some(source) = &source { + match resolve(source, &repo) { + Ok(found) => baseline = found, + // What the results are worth on their own outlives the comparison, + // so a baseline out of reach costs the changes, not the report. + Err(error) => eprintln!("{error:#}"), + } + } + + // The first run of a configuration has nothing to compare against, and CI + // carries on with the baseline it could not download. + baseline + .inputs + .retain(|baseline| has_results(&baseline.dir)); + if baseline.inputs.is_empty() { + baseline.commit = None; + if source.is_some() { + eprintln!("No baseline results found, reporting without a comparison"); + } + } + + if let Some(measured) = measured(&repo, &candidate, &baseline) { + print!("{measured}"); + } + + // A CI run covers every hypervisor and cpu vendor, one section each. + for candidate in &candidate.inputs { + let label = candidate.label.as_deref(); + let markdown = report( + &args, + config.as_ref(), + thresholds, + summary_limit, + &candidate.dir, + baseline_for(&baseline.inputs, candidate), + title(args.collapsible.as_deref(), label), + ) + .await?; + print!("{markdown}"); + } + + if reproduce_wanted { + print!("{}", reproduce(&args, &candidate, &baseline)); + } + + Ok(()) +} + +/// The command that reports these same results again. +/// +/// What was asked for moves: the last run of a pull request is whichever ran +/// most recently, and where it branched changes when it is rebased. Naming the +/// runs that answered holds the report still. +fn reproduce(args: &BenchReportArgs, candidate: &Origin, baseline: &Origin) -> String { + let mut command = format!("cargo ci bench-report --candidate {}", candidate.pinned); + + if !baseline.inputs.is_empty() { + command.push_str(&format!(" --baseline {}", baseline.pinned)); + } + + if let Some(config) = &args.config_file { + command.push_str(&format!(" --config-file {}", config.display())); + } + + format!("\nReported by `{command}`.\n") +} + +/// Where a change is worth reporting. The command line answers first, then the +/// config file, then the renderer. +fn thresholds( + args: &BenchReportArgs, + config: Option<&BenchConfig>, +) -> criterion_markdown::ChangeThresholds { + let from_config = |read: fn(&BenchConfig) -> Option| config.and_then(read); + let mut thresholds = criterion_markdown::ChangeThresholds::default(); + + if let Some(ratio) = args.improvement.or_else(|| from_config(|c| c.improvement)) { + thresholds = thresholds.improvement_ratio(ratio); + } + if let Some(ratio) = args + .strong_improvement + .or_else(|| from_config(|c| c.strong_improvement)) + { + thresholds = thresholds.strong_improvement_ratio(ratio); + } + if let Some(ratio) = args.regression.or_else(|| from_config(|c| c.regression)) { + thresholds = thresholds.regression_ratio(ratio); + } + + thresholds +} + +/// Say which commits the report covers, so a reader can tell what they are +/// looking at without knowing how it was asked for. +fn measured(repo: &str, candidate: &Origin, baseline: &Origin) -> Option { + let link = |sha: &String| { + let short = sha.get(..12).unwrap_or(sha); + format!("[`{short}`](https://github.com/{repo}/commit/{sha})") + }; + + let mut lines = format!("Measured commit: {}", candidate.commit.as_ref().map(link)?); + if let Some(baseline) = baseline.commit.as_ref().map(link) { + lines.push_str(&format!("\nBaseline commit: {baseline}")); + } + lines.push_str("\n\n"); + + Some(lines) +} + +/// Locate the results `source` points at. +fn resolve(source: &Source, repo: &str) -> Result { + let run = match source { + Source::Dir(dir) => { + return Ok(Origin { + commit: None, + pinned: dir.display().to_string(), + inputs: vec![Input { + label: None, + host: host_of(dir)?, + dir: dir.clone(), + }], + }); + } + Source::Release(tag) => { + eprintln!("Fetching release {tag} of {repo}"); + return Ok(Origin { + commit: remote::commit_sha(repo, tag).ok(), + pinned: format!("release:{tag}"), + inputs: inputs(remote::fetch_release(repo, tag, Path::new(RUN_CACHE))?)?, + }); + } + Source::Run(run) => *run, + Source::PullRequest(pull_request) => remote::latest_run_for(repo, *pull_request)?, + Source::Commit(commit) => remote::run_at(repo, commit)?, + Source::BaseOf(pull_request) => { + let commit = remote::merge_base_of(repo, *pull_request)?; + eprintln!("Pull request {pull_request} branched at {}", &commit[..12]); + remote::run_at(repo, &commit)? + } + }; + + eprintln!("Fetching run {run} of {repo}"); + Ok(Origin { + // A report reads the same without it, so it is not worth failing over. + commit: remote::run_commit(repo, run).ok(), + pinned: format!("run:{run}"), + inputs: inputs(remote::fetch(repo, run, Path::new(RUN_CACHE))?)?, + }) +} + +/// Read what each set of results says about the machine that took them. +fn inputs(results: Vec) -> Result> { + results + .into_iter() + .map(|results| { + Ok(Input { + // The artifact name says what produced it, so trust it over + // anything an older run left without a hypervisor recorded. + host: Identity::from_label(&results.label) + .map(Some) + .map_or_else(|| host_of(&results.dir), Ok)?, + label: Some(results.label), + dir: results.dir, + }) + }) + .collect() +} + +/// What the run in `dir` recorded about the machine it ran on. +fn host_of(dir: &Path) -> Result> { + Ok(manifest::read(dir)?.and_then(|manifest| Identity::from_host(&manifest.host))) +} + +/// Whether `dir` holds anything to report. Criterion writes nothing until a +/// benchmark runs, so an empty directory is one that never did. +fn has_results(dir: &Path) -> bool { + fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_some()) +} + +/// The baseline to compare `candidate` against. +fn baseline_for<'a>(baselines: &'a [Input], candidate: &Input) -> Option<&'a Path> { + match baselines { + [] => None, + // Nothing to tell apart, so a lone baseline stands in for whatever it + // is compared against. + [only] if only.host.is_none() || candidate.host.is_none() => Some(&only.dir), + _ => { + let host = candidate.host.as_ref()?; + let what = candidate + .label + .as_deref() + .map_or_else(|| "these results".to_string(), describe); + let mut found = baselines.iter().filter(|baseline| { + baseline + .host + .as_ref() + .is_some_and(|other| host.matches(other)) + }); + + match (found.next(), found.next()) { + (Some(baseline), None) => Some(baseline.dir.as_path()), + // Results that do not say which hypervisor produced them can + // fit more than one configuration. + (Some(_), Some(_)) => { + eprintln!("Several baselines fit {what}, reporting them alone"); + None + } + _ => { + eprintln!("Nothing to compare {what} against, reporting them alone"); + None + } + } + } + } +} + +/// Name the report after the configuration it covers. +fn title(summary: Option<&str>, label: Option<&str>) -> Option { + match (summary, label.map(describe)) { + (Some(summary), Some(label)) => Some(format!("{summary} {label}")), + (Some(summary), None) => Some(summary.to_string()), + (None, label) => label, + } +} + +/// Name a configuration the way the workflow that measured it does, turning +/// `Linux_kvm_amd` into `kvm / amd (Linux)`. +fn describe(label: &str) -> String { + let named = label + .split_once('_') + .and_then(|(os, rest)| Some((os, rest.rsplit_once('_')?))); + + match named { + Some((os, (hypervisor, vendor))) => format!("{hypervisor} / {vendor} ({os})"), + None => label.to_string(), + } +} + +/// Render the results in `dir`. +async fn report( + args: &BenchReportArgs, + config: Option<&BenchConfig>, + thresholds: criterion_markdown::ChangeThresholds, + summary_limit: Option, + dir: &Path, + baseline_root: Option<&Path>, + title: Option, +) -> Result { + let mut benchmarks = discover_benchmarks(args, dir).await?; + + if let Some(config) = config { + benchmarks = config.select(benchmarks)?; + } + + let mut renderer = criterion_markdown::Renderer::new(dir) + .benchmarks(benchmarks) + .change_thresholds(thresholds); + + if let Some(limit) = summary_limit { + renderer = renderer.summary_limit(limit); + } + + // Criterion keeps the last run of a directory in `new` and the one before + // it in `base`, so another directory is compared through its own last run. + if let Some(root) = baseline_root { + renderer = renderer.baseline_root(root).baseline("new"); + } + + // The summary doubles as the title of a collapsed report. + if let Some(title) = title { + renderer = renderer.title(title).collapsible(true); + } + + renderer.render() +} + +/// Benchmark ids for the results being reported. +/// +/// A run records what it measured, so prefer that: listing the binaries builds +/// them and describes the current checkout rather than the run in hand, which +/// differ whenever results come from elsewhere. Explicit binaries or bench args +/// ask for the binaries, and older results carry no manifest. +async fn discover_benchmarks(args: &BenchReportArgs, dir: &Path) -> Result> { + if args.binary.is_empty() && args.bench_args.is_empty() { + if let Some(manifest) = manifest::read(dir)? { + return Ok(manifest.benchmarks); + } + } + + let mut swarm = CriterionSwarm::builder(); + + if !args.binary.is_empty() { + swarm = swarm.binaries(&args.binary); + } + + for arg in &args.bench_args { + swarm = swarm.bench_arg(arg); + } + + let discovered = swarm + .output(NoopReporter) + .prepare() + .await + .context("Failed to discover benchmarks")?; + + Ok(discovered + .benchmarks() + .into_iter() + .map(str::to_string) + .collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn names_a_configuration_after_its_artifact() { + assert_eq!(describe("Linux_kvm_amd"), "kvm / amd (Linux)"); + assert_eq!( + describe("Windows_hyperv-ws2025_intel"), + "hyperv-ws2025 / intel (Windows)" + ); + } + + #[test] + fn keeps_a_name_it_cannot_read() { + assert_eq!(describe("whatever"), "whatever"); + assert_eq!(describe("Linux_kvm"), "Linux_kvm"); + } + + #[test] + fn titles_carry_both_the_summary_and_the_configuration() { + assert_eq!(title(None, None), None); + assert_eq!(title(Some("PR 1529"), None).as_deref(), Some("PR 1529")); + assert_eq!( + title(None, Some("Linux_kvm_amd")).as_deref(), + Some("kvm / amd (Linux)") + ); + assert_eq!( + title(Some("PR 1529"), Some("Linux_kvm_amd")).as_deref(), + Some("PR 1529 kvm / amd (Linux)") + ); + } +} diff --git a/src/hyperlight_ci/src/config.rs b/src/hyperlight_ci/src/config.rs new file mode 100644 index 0000000000..feb56971be --- /dev/null +++ b/src/hyperlight_ci/src/config.rs @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2025 The Hyperlight Authors. +//! The benchmark report configuration shared by the `bench` and `bench-report` subcommands. + +use std::path::Path; + +use anyhow::{Context, Result, bail}; +use regex::RegexSet; +use serde::Deserialize; + +/// Unknown keys are rejected so a stale key cannot silently disable filtering. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ConfigFile { + #[serde(default)] + allowlist: Vec, + #[serde(default)] + denylist: Vec, + improvement: Option, + strong_improvement: Option, + regression: Option, + repo: Option, + summary_limit: Option, + reproduce: Option, +} + +/// Benchmark id patterns selecting which results are reported. +#[derive(Debug)] +pub struct BenchConfig { + allow: RegexSet, + deny: RegexSet, + /// Where a change is worth reporting, when the file says. + pub improvement: Option, + pub strong_improvement: Option, + pub regression: Option, + /// Which repository the runs belong to. + pub repo: Option, + /// How many changes to call out before the tables. + pub summary_limit: Option, + /// Whether a report says how to ask for it again. + pub reproduce: Option, +} + +impl BenchConfig { + /// Read `allowlist` and `denylist` pattern arrays from a TOML file. + pub fn load(path: &Path) -> Result { + let text = std::fs::read_to_string(path) + .with_context(|| format!("Failed to read benchmark config {}", path.display()))?; + + Self::parse(&text).with_context(|| format!("Invalid benchmark config {}", path.display())) + } + + fn parse(text: &str) -> Result { + let file: ConfigFile = toml::from_str(text)?; + + Ok(Self { + allow: RegexSet::new(&file.allowlist)?, + deny: RegexSet::new(&file.denylist)?, + improvement: file.improvement, + strong_improvement: file.strong_improvement, + regression: file.regression, + repo: file.repo, + summary_limit: file.summary_limit, + reproduce: file.reproduce, + }) + } + + /// Keep the selected benchmarks, rejecting allowlist patterns that match nothing. + /// + /// Empty lists keep every benchmark. A denylist pattern matching nothing is + /// accepted, because a benchmark may be absent on some platforms. + pub fn select(&self, benchmarks: impl IntoIterator) -> Result> { + let mut used = vec![false; self.allow.len()]; + let mut selected = Vec::new(); + + for benchmark in benchmarks { + let matches = self.allow.matches(&benchmark); + for index in matches.iter() { + used[index] = true; + } + + let allowed = self.allow.is_empty() || matches.matched_any(); + if allowed && !self.deny.is_match(&benchmark) { + selected.push(benchmark); + } + } + + let stale: Vec<&str> = self + .allow + .patterns() + .iter() + .zip(&used) + .filter(|(_, used)| !**used) + .map(|(pattern, _)| pattern.as_str()) + .collect(); + + if !stale.is_empty() { + bail!( + "Benchmark allowlist patterns match no benchmark: {}", + stale.join(", ") + ); + } + + if selected.is_empty() && self.filters() { + bail!("Benchmark config excludes every benchmark"); + } + + Ok(selected) + } + + /// Whether the config restricts the reported benchmarks at all. + fn filters(&self) -> bool { + !self.allow.is_empty() || !self.deny.is_empty() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn benchmarks(names: &[&str]) -> Vec { + names.iter().map(|name| name.to_string()).collect() + } + + #[test] + fn reads_the_thresholds_when_given() { + let config = BenchConfig::parse("improvement = 1.5\nregression = 0.5").unwrap(); + + assert_eq!(config.improvement, Some(1.5)); + assert_eq!(config.regression, Some(0.5)); + assert_eq!(config.strong_improvement, None); + } + + #[test] + fn leaves_the_thresholds_alone_when_absent() { + let config = BenchConfig::parse(r#"allowlist = ["^sandboxes/"]"#).unwrap(); + + assert_eq!(config.improvement, None); + assert_eq!(config.strong_improvement, None); + assert_eq!(config.regression, None); + } + + #[test] + fn selects_only_allowlisted_benchmarks() { + let config = BenchConfig::parse(r#"allowlist = ["^sandboxes/", "^guest_calls/"]"#).unwrap(); + let selected = config + .select(benchmarks(&[ + "sandboxes/create", + "snapshots/save", + "guest_calls/small", + ])) + .unwrap(); + + assert_eq!(selected, ["sandboxes/create", "guest_calls/small"]); + } + + #[test] + fn denylist_subtracts_from_the_allowlist() { + let config = BenchConfig::parse( + r#" + allowlist = ["^sandboxes/"] + denylist = ["^sandboxes/noisy"] + "#, + ) + .unwrap(); + + let selected = config + .select(benchmarks(&["sandboxes/create", "sandboxes/noisy_case"])) + .unwrap(); + + assert_eq!(selected, ["sandboxes/create"]); + } + + #[test] + fn empty_allowlist_keeps_everything_not_denied() { + let config = BenchConfig::parse(r#"denylist = ["^snapshots/"]"#).unwrap(); + let selected = config + .select(benchmarks(&["sandboxes/create", "snapshots/save"])) + .unwrap(); + + assert_eq!(selected, ["sandboxes/create"]); + } + + #[test] + fn rejects_allowlist_patterns_matching_nothing() { + let config = + BenchConfig::parse(r#"allowlist = ["^sandboxes/", "^renamed_away/"]"#).unwrap(); + let error = config + .select(benchmarks(&["sandboxes/create"])) + .unwrap_err() + .to_string(); + + assert!(error.contains("^renamed_away/"), "{error}"); + assert!(!error.contains("^sandboxes/"), "{error}"); + } + + #[test] + fn accepts_denylist_patterns_matching_nothing() { + let config = BenchConfig::parse( + r#" + allowlist = ["^sandboxes/"] + denylist = ["^windows_only/"] + "#, + ) + .unwrap(); + + let selected = config.select(benchmarks(&["sandboxes/create"])).unwrap(); + assert_eq!(selected, ["sandboxes/create"]); + } + + #[test] + fn rejects_a_config_that_excludes_everything() { + let config = BenchConfig::parse( + r#" + allowlist = ["^sandboxes/"] + denylist = ["^sandboxes/"] + "#, + ) + .unwrap(); + + let error = config + .select(benchmarks(&["sandboxes/create"])) + .unwrap_err() + .to_string(); + + assert!(error.contains("excludes every benchmark"), "{error}"); + } + + #[test] + fn config_without_keys_keeps_every_benchmark() { + let config = BenchConfig::parse("").unwrap(); + let selected = config + .select(benchmarks(&["sandboxes/create", "snapshots/save"])) + .unwrap(); + + assert_eq!(selected, ["sandboxes/create", "snapshots/save"]); + } + + #[test] + fn empty_lists_keep_every_benchmark() { + let config = BenchConfig::parse("allowlist = []\ndenylist = []").unwrap(); + let selected = config + .select(benchmarks(&["sandboxes/create", "snapshots/save"])) + .unwrap(); + + assert_eq!(selected, ["sandboxes/create", "snapshots/save"]); + } + + #[test] + fn rejects_unknown_keys() { + let error = BenchConfig::parse(r#"patterns = ["^sandboxes/"]"#) + .unwrap_err() + .to_string(); + assert!(error.contains("unknown field"), "{error}"); + } + + #[test] + fn rejects_invalid_pattern() { + let error = BenchConfig::parse(r#"allowlist = ["^sandboxes/("]"#) + .unwrap_err() + .to_string(); + assert!(error.contains("regex parse error"), "{error}"); + } +} diff --git a/src/hyperlight_ci/src/main.rs b/src/hyperlight_ci/src/main.rs new file mode 100644 index 0000000000..5c37698e83 --- /dev/null +++ b/src/hyperlight_ci/src/main.rs @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2025 The Hyperlight Authors. +mod ballast; +mod bench; +mod bench_report; +mod config; +mod manifest; +mod remote; + +use clap::{Parser, Subcommand}; + +#[derive(Parser)] +#[command( + name = "hyperlight-ci", + about = "Hyperlight's CI and development tools" +)] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand)] +enum Commands { + /// Run benchmarks using the benchmark binary directly + Bench(bench::BenchArgs), + /// Generate a markdown table from existing criterion benchmark results + BenchReport(bench_report::BenchReportArgs), +} + +#[tokio::main(flavor = "current_thread")] +async fn main() -> anyhow::Result<()> { + let cli = Cli::parse(); + match cli.command { + Commands::Bench(args) => bench::run(args).await, + Commands::BenchReport(args) => bench_report::run(args).await, + } +} diff --git a/src/hyperlight_ci/src/manifest.rs b/src/hyperlight_ci/src/manifest.rs new file mode 100644 index 0000000000..05e6d6af62 --- /dev/null +++ b/src/hyperlight_ci/src/manifest.rs @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2025 The Hyperlight Authors. +//! A record of what a benchmark run measured, and on what. + +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; +use std::{env, fs}; + +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; + +/// Name of the manifest within the criterion results directory. +const FILE_NAME: &str = "benchmarks.json"; + +/// Written alongside the criterion results, so a run can be interpreted without +/// the benchmark binaries that produced it. +/// +/// Criterion records an id per result directory but nothing about the run as a +/// whole. Results also accumulate: a directory carries benchmarks that no +/// longer exist, indistinguishable from the ones just measured. This lists what +/// the run actually covered. +#[derive(Serialize, Deserialize)] +pub(crate) struct Manifest { + /// Seconds since the Unix epoch. Criterion timestamps nothing, and archived + /// results lose their file times. + timestamp: u64, + pub host: Host, + pub benchmarks: Vec, +} + +#[derive(Serialize, Deserialize)] +pub(crate) struct Host { + pub os: String, + pub arch: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub logical_cpus: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cpu_vendor: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cpu_model: Option, + /// The hypervisor hyperlight would use here, `kvm` and so on. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub hypervisor: Option, +} + +/// Where criterion keeps its results. +fn criterion_dir() -> PathBuf { + env::var_os("CRITERION_HOME") + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from("target").join("criterion")) +} + +#[cfg(target_os = "linux")] +fn cpu_vendor_and_model() -> (Option, Option) { + let Ok(text) = fs::read_to_string("/proc/cpuinfo") else { + return (None, None); + }; + let field = |key: &str| { + text.lines() + .find_map(|l| l.split_once(':').filter(|(k, _)| k.trim() == key)) + .map(|(_, v)| v.trim().to_string()) + }; + (field("vendor_id"), field("model name")) +} + +#[cfg(target_os = "windows")] +fn cpu_vendor_and_model() -> (Option, Option) { + // e.g. "Intel64 Family 6 Model 154 Stepping 3, GenuineIntel" + let model = env::var("PROCESSOR_IDENTIFIER").ok(); + let vendor = model + .as_deref() + .and_then(|m| m.rsplit_once(',')) + .map(|(_, v)| v.trim().to_string()); + (vendor, model) +} + +#[cfg(not(any(target_os = "linux", target_os = "windows")))] +fn cpu_vendor_and_model() -> (Option, Option) { + (None, None) +} + +/// Kvm and mshv cannot both be present, so the device that exists names the +/// hypervisor hyperlight would pick. +#[cfg(target_os = "linux")] +fn hypervisor() -> Option { + [("/dev/kvm", "kvm"), ("/dev/mshv", "mshv")] + .into_iter() + .find(|(device, _)| Path::new(device).exists()) + .map(|(_, name)| name.to_string()) +} + +#[cfg(target_os = "windows")] +fn hypervisor() -> Option { + Some("whp".to_string()) +} + +#[cfg(target_os = "macos")] +fn hypervisor() -> Option { + Some("hvf".to_string()) +} + +#[cfg(not(any(target_os = "linux", target_os = "windows", target_os = "macos")))] +fn hypervisor() -> Option { + None +} + +/// Record `benchmarks` as the contents of the run about to start. +pub(crate) fn write(benchmarks: impl IntoIterator) -> Result<()> { + let (cpu_vendor, cpu_model) = cpu_vendor_and_model(); + let mut benchmarks: Vec = benchmarks.into_iter().collect(); + benchmarks.sort(); + + let manifest = Manifest { + timestamp: SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or_default(), + host: Host { + os: env::consts::OS.to_string(), + arch: env::consts::ARCH.to_string(), + logical_cpus: std::thread::available_parallelism().ok().map(Into::into), + cpu_vendor, + cpu_model, + hypervisor: hypervisor(), + }, + benchmarks, + }; + + let dir = criterion_dir(); + fs::create_dir_all(&dir).with_context(|| format!("Failed to create {}", dir.display()))?; + let path = dir.join(FILE_NAME); + let json = serde_json::to_string_pretty(&manifest)?; + fs::write(&path, json).with_context(|| format!("Failed to write {}", path.display())) +} + +/// What a run recorded, or `None` when it left no manifest. +pub(crate) fn read(dir: &Path) -> Result> { + let path = dir.join(FILE_NAME); + let Ok(text) = fs::read_to_string(&path) else { + return Ok(None); + }; + + serde_json::from_str(&text) + .map(Some) + .with_context(|| format!("Failed to parse {}", path.display())) +} diff --git a/src/hyperlight_ci/src/remote.rs b/src/hyperlight_ci/src/remote.rs new file mode 100644 index 0000000000..0606d3bb0b --- /dev/null +++ b/src/hyperlight_ci/src/remote.rs @@ -0,0 +1,438 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2025 The Hyperlight Authors. +//! Benchmark results taken from a CI run rather than this machine. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use anyhow::{Context, Result, bail}; +use serde::Deserialize; + +/// Artifacts holding a criterion directory are named after the configuration +/// that produced them, `benchmarks_Linux_kvm_amd` and so on. +const ARTIFACT_PREFIX: &str = "benchmarks_"; + +/// How far back to look for a run that still has its benchmark artifacts. +/// A day of the default branch is one run, so this spans how long they are kept. +const RUNS_SEARCHED: usize = 90; + +/// Where benchmarks of the default branch come from. Pull requests benchmark +/// far more often, but never the branch they merge into. +const BASELINE_WORKFLOW: &str = "DailyBenchmarks.yml"; + +/// What a release calls the results it carries. +const ARCHIVE_SUFFIX: &str = ".tar.gz"; + +/// How many times to ask for a listing before taking it at its word. +const LISTINGS: usize = 5; + +/// Marks a download that finished. Artifacts hold nothing every run is bound +/// to leave behind, and an interrupted one leaves the directory half written. +const DOWNLOADED: &str = ".downloaded"; + +/// One configuration's results, and where they were unpacked. +pub(crate) struct Results { + /// The configuration that produced them, `Linux_kvm_amd` and so on. + pub label: String, + pub dir: PathBuf, +} + +#[derive(Deserialize)] +struct Artifact { + name: String, + expired: bool, +} + +#[derive(Deserialize)] +struct ArtifactList { + artifacts: Vec, +} + +#[derive(Deserialize)] +struct Run { + #[serde(rename = "databaseId")] + id: u64, + #[serde(rename = "headSha")] + head_sha: String, + #[serde(rename = "createdAt")] + created_at: String, +} + +/// Run `gh` and hand back its stdout. +fn gh(args: &[&str]) -> Result> { + let output = Command::new("gh") + .args(args) + .output() + .context("Failed to run gh. The GitHub CLI provides the run artifacts")?; + + if !output.status.success() { + bail!( + "gh {} failed: {}", + args.join(" "), + String::from_utf8_lossy(&output.stderr).trim() + ); + } + + Ok(output.stdout) +} + +/// Which repository to read, either `owner/name` or whichever one a git +/// remote points at, `remote:origin`. +pub(crate) fn repository(value: &str) -> Result { + let Some(remote) = value.strip_prefix("remote:") else { + return Ok(value.to_string()); + }; + + let output = Command::new("git") + .args(["remote", "get-url", remote]) + .output() + .context("Failed to run git")?; + + if !output.status.success() { + bail!( + "Failed to read the url of remote {remote}: {}", + String::from_utf8_lossy(&output.stderr).trim() + ); + } + + let url = String::from_utf8_lossy(&output.stdout); + owner_and_name(&url) + .with_context(|| format!("Remote {remote} names no repository: {}", url.trim())) +} + +/// The owner and name a clone url ends with, however it spells the host. +fn owner_and_name(url: &str) -> Option { + let url = url.trim().trim_end_matches('/'); + let url = url.strip_suffix(".git").unwrap_or(url); + + let mut parts = url.rsplit(['/', ':']); + let name = parts.next()?; + let owner = parts.next()?; + + (!name.is_empty() && !owner.is_empty()).then(|| format!("{owner}/{name}")) +} + +/// Names of the benchmark artifacts a run still holds. +fn artifacts(repo: &str, run: u64) -> Result> { + let path = format!("repos/{repo}/actions/runs/{run}/artifacts"); + let list: ArtifactList = serde_json::from_slice(&gh(&["api", &path])?) + .with_context(|| format!("Failed to read the artifacts of run {run}"))?; + + let mut names: Vec = list + .artifacts + .into_iter() + .filter(|a| !a.expired && a.name.starts_with(ARTIFACT_PREFIX)) + .map(|a| a.name) + .collect(); + names.sort(); + names.dedup(); + Ok(names) +} + +/// Runs matching `filter`, newest first. +/// +/// GitHub answers out of an index that takes a moment to warm, leaving the +/// most recent runs out of the first replies. Taking one at its word picks a +/// baseline months older than the one asked for, so ask until two replies +/// agree on the newest run and keep everything either of them saw. +fn runs(repo: &str, filter: &[&str]) -> Result> { + let limit = RUNS_SEARCHED.to_string(); + let mut seen: Vec = Vec::new(); + let mut newest = None; + + for _ in 0..LISTINGS { + let mut args = vec![ + "run", + "list", + "--repo", + repo, + "--limit", + &limit, + "--json", + "databaseId,headSha,createdAt", + ]; + args.extend_from_slice(filter); + + let listed: Vec = + serde_json::from_slice(&gh(&args)?).context("Failed to list the workflow runs")?; + + let latest = listed.first().map(|run| run.id); + seen.extend(listed); + + if latest.is_some() && latest == newest { + break; + } + newest = latest; + } + + seen.sort_by(|left, right| (&right.created_at, right.id).cmp(&(&left.created_at, left.id))); + seen.dedup_by_key(|run| run.id); + Ok(seen) +} + +/// The most recent run of `pull_request` that still has benchmark artifacts. +/// +/// The newest run is not always the one to report: a run can be cancelled by +/// the next push, or be recent enough that the benchmarks have not finished. +pub(crate) fn latest_run_for(repo: &str, pull_request: u64) -> Result { + let pr = pull_request.to_string(); + let branch = gh(&[ + "pr", + "view", + &pr, + "--repo", + repo, + "--json", + "headRefName", + "--jq", + ".headRefName", + ]) + .with_context(|| format!("Failed to find pull request {pull_request}"))?; + let branch = String::from_utf8_lossy(&branch).trim().to_string(); + + for run in runs(repo, &["--branch", &branch])? { + if !artifacts(repo, run.id)?.is_empty() { + return Ok(run.id); + } + } + + bail!("No run of {branch} still has benchmark artifacts") +} + +/// Resolve a sha, tag or branch to the commit it names. +pub(crate) fn commit_sha(repo: &str, commit: &str) -> Result { + let path = format!("repos/{repo}/commits/{commit}"); + let sha = gh(&["api", &path, "--jq", ".sha"]) + .with_context(|| format!("Failed to find commit {commit}"))?; + Ok(String::from_utf8_lossy(&sha).trim().to_string()) +} + +/// The commit a run measured. +pub(crate) fn run_commit(repo: &str, run: u64) -> Result { + let id = run.to_string(); + let sha = gh(&[ + "run", "view", &id, "--repo", repo, "--json", "headSha", "--jq", ".headSha", + ]) + .with_context(|| format!("Failed to find what run {run} measured"))?; + Ok(String::from_utf8_lossy(&sha).trim().to_string()) +} + +/// Whether `commit` is `ancestor` or was built on top of it. +fn descends_from(repo: &str, commit: &str, ancestor: &str) -> Result { + let path = format!("repos/{repo}/compare/{ancestor}...{commit}"); + let status = gh(&["api", &path, "--jq", ".status"])?; + Ok(matches!( + String::from_utf8_lossy(&status).trim(), + "identical" | "ahead" + )) +} + +/// The most recent benchmarks of the default branch taken at or before +/// `commit`. +/// +/// The branch is benchmarked daily rather than per commit, so the run that +/// measured `commit` itself rarely exists. Anything measured after it carries +/// changes the commit never had. +pub(crate) fn run_at(repo: &str, commit: &str) -> Result { + let commit = commit_sha(repo, commit)?; + // A cancelled run leaves some configurations unmeasured. + let runs = runs( + repo, + &["--workflow", BASELINE_WORKFLOW, "--status", "success"], + )?; + + for run in &runs { + if descends_from(repo, &commit, &run.head_sha)? && !artifacts(repo, run.id)?.is_empty() { + return Ok(run.id); + } + } + + bail!("No benchmarks taken at or before {commit} still have their artifacts") +} + +/// Where `pull_request` branched off the branch it targets. +pub(crate) fn merge_base_of(repo: &str, pull_request: u64) -> Result { + let pr = pull_request.to_string(); + let refs = gh(&[ + "pr", + "view", + &pr, + "--repo", + repo, + "--json", + "baseRefName,headRefOid", + "--jq", + ".baseRefName + \" \" + .headRefOid", + ]) + .with_context(|| format!("Failed to find pull request {pull_request}"))?; + + let refs = String::from_utf8_lossy(&refs); + let Some((base, head)) = refs.trim().split_once(' ') else { + bail!("Pull request {pull_request} has no branch to compare against"); + }; + + let path = format!("repos/{repo}/compare/{base}...{head}"); + let sha = gh(&["api", &path, "--jq", ".merge_base_commit.sha"]) + .with_context(|| format!("Failed to find where pull request {pull_request} branched"))?; + Ok(String::from_utf8_lossy(&sha).trim().to_string()) +} + +/// Fetch every configuration's results from `run`, reusing what is already on +/// disk. Artifacts are immutable, so a run downloads once. +pub(crate) fn fetch(repo: &str, run: u64, cache: &Path) -> Result> { + let names = artifacts(repo, run)?; + if names.is_empty() { + bail!("Run {run} has no benchmark artifacts. They may have expired"); + } + + let run_dir = cache.join(run.to_string()); + let mut results = Vec::new(); + + for name in names { + let label = name[ARTIFACT_PREFIX.len()..].to_string(); + let dir = run_dir.join(&label); + + if !dir.join(DOWNLOADED).exists() { + if dir.exists() { + fs::remove_dir_all(&dir) + .with_context(|| format!("Failed to clear {}", dir.display()))?; + } + fs::create_dir_all(&dir) + .with_context(|| format!("Failed to create {}", dir.display()))?; + + let (id, out) = (run.to_string(), dir.display().to_string()); + gh(&[ + "run", "download", &id, "--repo", repo, "-n", &name, "-D", &out, + ]) + .with_context(|| format!("Failed to download {name}"))?; + + fs::write(dir.join(DOWNLOADED), []) + .with_context(|| format!("Failed to mark {} downloaded", dir.display()))?; + } + + results.push(Results { label, dir }); + } + + Ok(results) +} + +/// Names of the benchmark archives a release carries. +fn assets(repo: &str, tag: &str) -> Result> { + let names = gh(&[ + "release", + "view", + tag, + "--repo", + repo, + "--json", + "assets", + "--jq", + ".assets[].name", + ]) + .with_context(|| format!("Failed to find release {tag}"))?; + + let mut names: Vec = String::from_utf8_lossy(&names) + .lines() + .filter(|name| name.starts_with(ARTIFACT_PREFIX) && name.ends_with(ARCHIVE_SUFFIX)) + .map(str::to_string) + .collect(); + names.sort(); + names.dedup(); + Ok(names) +} + +/// Fetch every configuration's results from the release tagged `tag`. +/// +/// A release carries what it measured for as long as it exists, which is past +/// the day the workflow artifacts of the same run are swept away. +pub(crate) fn fetch_release(repo: &str, tag: &str, cache: &Path) -> Result> { + let names = assets(repo, tag)?; + if names.is_empty() { + bail!("Release {tag} carries no benchmark results"); + } + + let release_dir = cache.join(format!("release-{tag}")); + let mut results = Vec::new(); + + for name in names { + let label = name[ARTIFACT_PREFIX.len()..name.len() - ARCHIVE_SUFFIX.len()].to_string(); + let dir = release_dir.join(&label); + + if !dir.join(DOWNLOADED).exists() { + if dir.exists() { + fs::remove_dir_all(&dir) + .with_context(|| format!("Failed to clear {}", dir.display()))?; + } + fs::create_dir_all(&dir) + .with_context(|| format!("Failed to create {}", dir.display()))?; + + let out = dir.display().to_string(); + gh(&[ + "release", "download", tag, "--repo", repo, "-p", &name, "-D", &out, + ]) + .with_context(|| format!("Failed to download {name}"))?; + + // The archive holds the criterion directory under a name of its own. + let archive = dir.join(&name); + unpack(&archive, &dir)?; + fs::remove_file(&archive) + .with_context(|| format!("Failed to remove {}", archive.display()))?; + + fs::write(dir.join(DOWNLOADED), []) + .with_context(|| format!("Failed to mark {} downloaded", dir.display()))?; + } + + results.push(Results { label, dir }); + } + + Ok(results) +} + +/// Unpack `archive` into `into`, dropping the directory it wraps everything in. +fn unpack(archive: &Path, into: &Path) -> Result<()> { + let status = Command::new("tar") + .arg("-xzf") + .arg(archive) + .arg("-C") + .arg(into) + .arg("--strip-components=1") + .status() + .context("Failed to run tar. It unpacks the results a release carries")?; + + if !status.success() { + bail!("Failed to unpack {}", archive.display()); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_the_repository_a_clone_url_ends_with() { + for url in [ + "git@github.com:hyperlight-dev/hyperlight.git", + "https://github.com/hyperlight-dev/hyperlight.git", + "https://github.com/hyperlight-dev/hyperlight", + "ssh://git@github.com/hyperlight-dev/hyperlight.git", + " git@github.com:hyperlight-dev/hyperlight.git\n", + ] { + assert_eq!( + owner_and_name(url).as_deref(), + Some("hyperlight-dev/hyperlight"), + "{url}" + ); + } + } + + #[test] + fn keeps_a_repository_named_outright() { + assert_eq!( + repository("hyperlight-dev/hyperlight").unwrap(), + "hyperlight-dev/hyperlight" + ); + } +} diff --git a/src/hyperlight_host/examples/ballast/main.rs b/src/hyperlight_host/examples/ballast/main.rs new file mode 100644 index 0000000000..2d96fdcc17 --- /dev/null +++ b/src/hyperlight_host/examples/ballast/main.rs @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2025 The Hyperlight Authors. +//! Holds one sandbox alive and then idles. +//! +//! A vCPU created without an in-kernel LAPIC bumps the kernel's +//! `kvm_has_noapic_vcpu` static key, and teardown drops it again. Each +//! transition through zero patches kernel text, which IPIs every core. +//! Keeping one sandbox resident holds the count above zero, so a benchmark +//! that creates and drops sandboxes never crosses that boundary. +//! Parks after startup, so it consumes no CPU while a benchmark runs. + +use std::io::Read; + +use hyperlight_host::SandboxBuilder; + +fn main() -> hyperlight_host::Result<()> { + let _sandbox = + SandboxBuilder::from_file(hyperlight_testing::simple_guest_as_pathbuf()).build()?; + + // Readers wait for this line before starting to measure. + println!("ballast ready"); + + // Stdin reaches EOF when the parent closes it or dies, so the sandbox never + // outlives the run that asked for it. + let mut discard = Vec::new(); + let _ = std::io::stdin().read_to_end(&mut discard); + + Ok(()) +}