Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 144 additions & 0 deletions .github/actions/setup-self-hosted/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
name: Setup self-hosted Linux runner
description: >-
Install host tools that GitHub-hosted Ubuntu images include and the org
sm-standard runner images do not. Rust still invokes the system linker, and
aws-lc-sys (pulled in by the AWS SDK) builds with CMake. Docker is not
installed here: jobs that need a daemon use the dind-sm-standard-2 label.

inputs:
profile:
description: >-
Space-separated profiles: rust-build, python, openssl, curl, file,
archive-tools, github-cli, awscli.
required: true

runs:
using: composite
steps:
- name: Install host tools
shell: bash
env:
PROFILE: ${{ inputs.profile }}
run: |
set -euo pipefail

if ! command -v sudo >/dev/null 2>&1; then
echo "sudo is required to install host packages on the self-hosted runner" >&2
exit 1
fi

if [[ -z "${PROFILE// /}" ]]; then
echo "setup-self-hosted requires a profile" >&2
exit 1
fi

has() {
[[ " ${PROFILE} " == *" $1 "* ]]
}

for token in ${PROFILE}; do
case "${token}" in
rust-build|python|openssl|curl|file|archive-tools|github-cli|awscli) ;;
*)
echo "Unknown setup profile: ${token}" >&2
exit 1
;;
esac
done

packages=()
add_pkg() {
local pkg="$1"
local existing
if ((${#packages[@]} > 0)); then
for existing in "${packages[@]}"; do
if [[ "${existing}" == "${pkg}" ]]; then
return
fi
done
fi
packages+=("${pkg}")
}

# curl and CA certificates are how rustup and the GitHub CLI apt repo
# are fetched. GitHub-hosted images already have both.
if has rust-build || has curl || has github-cli; then
add_pkg ca-certificates
add_pkg curl
fi

# sm-standard images ship no C toolchain. rustc needs cc, and
# aws-lc-sys builds its C library with CMake. libssl-dev matches the
# org's other self-hosted lanes (git2/openssl build scripts).
if has rust-build; then
add_pkg build-essential
add_pkg cmake
add_pkg pkg-config
add_pkg libssl-dev
add_pkg perl
add_pkg git
fi

if has python; then
add_pkg python3
fi

if has openssl; then
add_pkg openssl
fi

# `file` is preinstalled on GitHub-hosted Ubuntu and is how the
# package job checks that release binaries are statically linked.
if has file; then
add_pkg file
fi

# Release archives are checksummed with shasum, which GitHub-hosted
# images provide via Perl. coreutils sha256sum is not a drop-in: the
# existing release script calls shasum.
if has archive-tools; then
add_pkg libdigest-sha-perl
fi

# Official AWS CLI v2, matching GitHub-hosted images. The Ubuntu awscli
# apt package is v1 and ignores AWS_REQUEST_CHECKSUM_CALCULATION, which
# the R2 upload steps set for CLI v2.23+.
if has awscli && ! command -v aws >/dev/null 2>&1; then
add_pkg ca-certificates
add_pkg curl
add_pkg unzip
fi

if ((${#packages[@]} > 0)); then
sudo DEBIAN_FRONTEND=noninteractive apt-get update
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y "${packages[@]}"
fi

if has awscli && ! command -v aws >/dev/null 2>&1; then
case "$(dpkg --print-architecture)" in
amd64) aws_url="https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" ;;
arm64) aws_url="https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" ;;
*)
echo "No AWS CLI v2 build for $(dpkg --print-architecture)" >&2
exit 1
;;
esac
aws_tmp="$(mktemp -d)"
curl -fsSL "${aws_url}" -o "${aws_tmp}/awscliv2.zip"
unzip -q "${aws_tmp}/awscliv2.zip" -d "${aws_tmp}"
sudo "${aws_tmp}/aws/install"
rm -rf "${aws_tmp}"
fi

if has github-cli && ! command -v gh >/dev/null 2>&1; then
# Ubuntu's own gh package is not always present on the minimal
# runner image. Use the upstream GitHub CLI apt repository.
sudo mkdir -p -m 755 /etc/apt/keyrings
curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg \
| sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg >/dev/null
sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" \
| sudo tee /etc/apt/sources.list.d/github-cli.list >/dev/null
sudo DEBIAN_FRONTEND=noninteractive apt-get update
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y gh
fi
110 changes: 98 additions & 12 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -1,22 +1,43 @@
name: CI

# Linux jobs run on the org's self-hosted runners (sm-standard-2 / sm-standard-4).
# cargo-deny-action is a container action, so Dependency Advisories uses the
# Docker-capable dind-sm-standard-2 label.
#
# The org has no self-hosted macOS or Windows runners. Those lanes execute
# `cargo test` / `cargo build` on the host OS; cross-compiling them on Linux
# would not run the tests and would not exercise the same linker. They stay on
# GitHub-hosted runners and are limited to workflow_dispatch.

on:
push:
branches: [ main, develop ]
pull_request:
branches: [ main, develop ]
workflow_dispatch:

env:
CARGO_TERM_COLOR: always
RUST_BACKTRACE: 1
# Self-hosted runner images may still bundle an older Node than current
# JavaScript actions require.
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"

permissions:
contents: read

# Cancel superseded PR runs. Push, schedule, and manual dispatch are left
# running so a new commit on a PR cannot abort a main or release-related run.
concurrency:
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
advisories:
name: Dependency Advisories
runs-on: ubuntu-latest
# Container action: the runner must have a Docker daemon. sm-standard-* does not.
runs-on: dind-sm-standard-2
timeout-minutes: 20
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
- uses: EmbarkStudios/cargo-deny-action@bb137d7af7e4fb67e5f82a49c4fce4fad40782fe # v2
Expand All @@ -25,9 +46,15 @@ jobs:

fmt:
name: Format Check
runs-on: ubuntu-latest
runs-on: sm-standard-2
timeout-minutes: 20
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
- name: Install host tools
uses: ./.github/actions/setup-self-hosted
with:
# rustup is fetched with curl. cargo fmt does not link.
profile: curl
- uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable
with:
components: rustfmt
Expand All @@ -36,9 +63,14 @@ jobs:

clippy:
name: Clippy
runs-on: ubuntu-latest
runs-on: sm-standard-4
timeout-minutes: 60
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
- name: Install host tools
uses: ./.github/actions/setup-self-hosted
with:
profile: rust-build
- uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable
with:
components: clippy
Expand All @@ -47,14 +79,37 @@ jobs:
run: cargo clippy --workspace --all-targets -- -D warnings

test:
name: Test (${{ matrix.os }})
runs-on: ${{ matrix.os }}
name: Test (${{ matrix.name }})
runs-on: ${{ matrix.runner }}
timeout-minutes: 60
# manual_only lanes are GitHub-hosted and must not run on push or pull_request.
# Compared as a string: a boolean false in a matrix is easy to misread as the
# truthy string "false" once it crosses an expression boundary.
if: ${{ matrix.manual_only != 'yes' || github.event_name == 'workflow_dispatch' }}
strategy:
fail-fast: false
matrix:
os: [ ubuntu-latest, macos-latest, windows-latest ]
include:
- name: linux
runner: sm-standard-4
manual_only: "no"
- name: macos-latest, manual
runner: macos-latest
manual_only: "yes"
- name: windows-latest, manual
runner: windows-latest
manual_only: "yes"
steps:
- name: Flag billed GitHub-hosted runner
if: matrix.manual_only == 'yes'
shell: bash
run: echo "::warning title=GitHub-hosted runner::Manual dispatch only. This job uses billed ${{ matrix.runner }} because the org has no self-hosted macOS or Windows runners. It does not run on push, pull_request, or schedule."
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
- name: Install host tools
if: runner.os == 'Linux'
uses: ./.github/actions/setup-self-hosted
with:
profile: rust-build
- uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
- name: Run tests
Expand All @@ -63,24 +118,49 @@ jobs:
run: cargo test --workspace

build:
name: Build (${{ matrix.os }})
runs-on: ${{ matrix.os }}
name: Build (${{ matrix.name }})
runs-on: ${{ matrix.runner }}
timeout-minutes: 90
if: ${{ matrix.manual_only != 'yes' || github.event_name == 'workflow_dispatch' }}
strategy:
fail-fast: false
matrix:
os: [ ubuntu-latest, macos-latest, windows-latest ]
include:
- name: linux
runner: sm-standard-4
manual_only: "no"
- name: macos-latest, manual
runner: macos-latest
manual_only: "yes"
- name: windows-latest, manual
runner: windows-latest
manual_only: "yes"
steps:
- name: Flag billed GitHub-hosted runner
if: matrix.manual_only == 'yes'
shell: bash
run: echo "::warning title=GitHub-hosted runner::Manual dispatch only. This job uses billed ${{ matrix.runner }} because the org has no self-hosted macOS or Windows runners. It does not run on push, pull_request, or schedule."
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
- name: Install host tools
if: runner.os == 'Linux'
uses: ./.github/actions/setup-self-hosted
with:
profile: rust-build
- uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
- name: Build
run: cargo build --workspace --release

doc:
name: Documentation
runs-on: ubuntu-latest
runs-on: sm-standard-4
timeout-minutes: 40
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
- name: Install host tools
uses: ./.github/actions/setup-self-hosted
with:
profile: rust-build
- uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
- name: Build documentation
Expand All @@ -91,7 +171,8 @@ jobs:
# Ensure protected files are not modified without proper process
protected-files:
name: Protected Files Check
runs-on: ubuntu-latest
runs-on: sm-standard-2
timeout-minutes: 10
if: github.event_name == 'pull_request'
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
Expand All @@ -104,9 +185,14 @@ jobs:

msrv:
name: Minimum Supported Rust Version
runs-on: ubuntu-latest
runs-on: sm-standard-4
timeout-minutes: 60
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
- name: Install host tools
uses: ./.github/actions/setup-self-hosted
with:
profile: rust-build
- uses: dtolnay/rust-toolchain@fa04a1451ff1842e2626ccb99004d0195b455a88 # master
with:
toolchain: "1.92"
Expand Down
12 changes: 11 additions & 1 deletion .github/workflows/docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,20 @@ on:
permissions:
contents: read

# workflow_run is an automatic trigger, so this job cannot use a GitHub-hosted
# runner. Image builds need a Docker daemon, which only the dind label provides.
concurrency:
group: ${{ github.workflow }}-${{ github.event.workflow_run.head_sha || github.sha }}
cancel-in-progress: false

env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"

jobs:
build-and-push:
if: github.event_name == 'workflow_dispatch' || (github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push')
runs-on: ubuntu-latest
runs-on: dind-sm-standard-2
timeout-minutes: 60

steps:
- name: Checkout code
Expand Down
Loading
Loading