From c338080675586f603b7ad005b11377282b91a605 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 24 Sep 2026 12:54:37 +0000 Subject: [PATCH] test: measure level-three workload benefit against exact databases --- .github/workflows/issue754-benefit.yml | 48 ++++++++++++++ promql-compliance/BENEFIT.md | 66 +++++++++++++++++++ promql-compliance/docker-compose.benefit.yml | 10 +++ promql-compliance/runner/Makefile | 11 ++++ .../runner/src/bin/benefit-runner.rs | 5 ++ 5 files changed, 140 insertions(+) create mode 100644 .github/workflows/issue754-benefit.yml create mode 100644 promql-compliance/BENEFIT.md create mode 100644 promql-compliance/docker-compose.benefit.yml create mode 100644 promql-compliance/runner/src/bin/benefit-runner.rs diff --git a/.github/workflows/issue754-benefit.yml b/.github/workflows/issue754-benefit.yml new file mode 100644 index 00000000..f79b1330 --- /dev/null +++ b/.github/workflows/issue754-benefit.yml @@ -0,0 +1,48 @@ +name: Issue 754 level-3 benefit + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + workflow_dispatch: + +permissions: + contents: read + +jobs: + benefit: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + with: + path: ASAPQuery-backend + - uses: actions/checkout@v4 + with: + repository: ProjectASAP/ASAPCollector + path: ASAPCollector + - uses: actions/checkout@v4 + with: + repository: ProjectASAP/asap_sketchlib + path: asap_sketchlib + - uses: dtolnay/rust-toolchain@stable + - name: Validate PR 761 automatic workload costing + working-directory: ASAPQuery-backend + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler + mkdir -p "$GITHUB_WORKSPACE/artifacts" + git rev-parse HEAD > "$GITHUB_WORKSPACE/artifacts/backend-revision.txt" + cargo test --locked -p control_plane --lib --bins + - name: Validate shared workload and SQL coverage + working-directory: ASAPQuery-backend/promql-compliance/runner + run: cargo test --locked -p promql-compliance + - name: Compare ASAPQuery with exact DB baselines + working-directory: ASAPQuery-backend/promql-compliance/runner + run: make benefit REPORT_DIR="$GITHUB_WORKSPACE/artifacts/reports" LOGS_DIR="$GITHUB_WORKSPACE/artifacts/logs" + - name: Upload benchmark and service logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: issue754-benefit-evidence + path: artifacts + if-no-files-found: warn diff --git a/promql-compliance/BENEFIT.md b/promql-compliance/BENEFIT.md new file mode 100644 index 00000000..564c0e66 --- /dev/null +++ b/promql-compliance/BENEFIT.md @@ -0,0 +1,66 @@ +# Issue 754 level-3 benefit test + +The Rust workspace binary `benefit-runner`, invoked by `make benefit` in `promql-compliance/runner` reuses the level-1 and level-2 +`issue-754.yaml` suite and dataset. It starts an isolated backend deployment, +Prometheus, VictoriaMetrics, and ClickHouse; writes identical samples; checks +backend results against Prometheus; checks the other exact baselines against +Prometheus; and then measures ten repeated instant-query executions after three +warmups per query and target. + +The ClickHouse SQL is generated by `runner/src/sql.rs` from the same ten case names. +Its float-counter rate path includes resets and Prometheus 3.5.0-style boundary +extrapolation. SQL output is compared with Prometheus before timing. The +benchmark reads cgroup-v2 CPU use and peak memory for each container and records +per-query p50/p95 latency samples. It fails if the backend does not improve CPU, +peak memory, and each query's p95 latency against every exact baseline. The full +JSON report and service logs are uploaded by the CI workflow even on failure. + +The checked-in CI fixture has eight series at 100 ms cadence and a one-minute +window. It is a deterministic acceptance point, not a scalability claim across +the full cardinality and time-range matrix in issue #754. Run larger matrix +points separately using the same suite and dataset schema and retain the same +semantic gates before interpreting performance. + +This test requires Rust/Cargo, protoc, Docker Compose, access to the repository's image build +dependencies, and cgroup v2. Run from `promql-compliance/runner`: + +```sh +make benefit +``` + +## PR 761 workload-cost gate + +This branch includes PR #761. The level-3 Rust runner builds a typed **unquoted** snapshot, invokes the +control-plane library directly and passes the same snapshot to backend startup. It derives source cadence, +input series count, sample volume and aggregate ingestion rate from the replay +dataset; query recurrence comes from the shared suite. Source cadence and query lookback retain +100 ms precision through backend admission. No synthetic complete +quotes or forced winner are inserted. + +Before the local-execution and correctness gates, the runner verifies every +priced candidate has an automatic resource breakdown, the selected component +keys exactly cover its manifest, resource weights reproduce the totals, ERP +provenance is explicit, and the selected candidate has the lowest available +cost. The saved `.plan.json` contains all candidate estimates, assumptions and +ERP IDs; the benchmark report links that plan. CI also runs the Rust workload +cost tests, including ERP precedence and fallback to analytical estimates. +The default level-3 fixture has no ERP artifact and exercises analytical costing; +ERP-priority coverage is currently in the Rust contract tests. + +Analytical planning estimates are not the measured benefit result. Costing uses +the declared 300-second planning horizon; timing measures the configured repeated +query trial batch. The original local-only, semantic and measured-benefit gates +remain mandatory: a correctly costed exact fallback still fails local-only +acceptance and does not establish a performance improvement. + +The inherited level-1 test still rejects the grouped temporal Sum plan and the +quantile-ratio exact fallback. These failures must be +resolved before this fixture can establish an end-to-end performance benefit; +the runner does not coarsen data cadence or weaken the local-execution gate. + +The three workspace binaries (`differential-runner`, `benefit-runner`, +`report-card`) and their contract/HTTP tests are Rust. CI uses Cargo and the +shared backend types for plan/cost validation. No Go tooling is required. +CPU deltas cover measured trials after warmups; memory is the container-lifetime +cgroup peak. SQL comparison checks source/group labels as well as values, and +fallback provenance is checked again during timed backend queries. diff --git a/promql-compliance/docker-compose.benefit.yml b/promql-compliance/docker-compose.benefit.yml new file mode 100644 index 00000000..c40b59d6 --- /dev/null +++ b/promql-compliance/docker-compose.benefit.yml @@ -0,0 +1,10 @@ +services: + clickhouse: + image: clickhouse/clickhouse-server:25.8 + environment: + CLICKHOUSE_SKIP_USER_SETUP: "1" + ports: ["${CLICKHOUSE_PORT:-18123}:8123"] + victoria: + image: victoriametrics/victoria-metrics:v1.126.0 + command: ["-retentionPeriod=1d", "-search.maxStalenessInterval=5m"] + ports: ["${VICTORIA_PORT:-18428}:8428"] diff --git a/promql-compliance/runner/Makefile b/promql-compliance/runner/Makefile index 4b9befe9..391824ee 100644 --- a/promql-compliance/runner/Makefile +++ b/promql-compliance/runner/Makefile @@ -30,3 +30,14 @@ run-all: echo "Running $$name"; \ if ! cargo run --locked -p promql-compliance --bin differential-runner -- --dataset "$$dataset" --suite "$$suite" --reference-url "$(REFERENCE_URL)" --test-url "$(BACKEND_URL)" --compose-file "$(COMPOSE_FILE)" --logs-dir "$(LOGS_DIR)/$$name" --output "$(REPORT_DIR)/$$name.json"; then result=1; fi; \ done; cargo run --locked -p promql-compliance --bin report-card -- --reports-dir "$(REPORT_DIR)"; exit $$result + +.PHONY: benefit +benefit: + mkdir -p "$(REPORT_DIR)" "$(LOGS_DIR)" + cargo run --locked -p promql-compliance --bin benefit-runner -- \ + --dataset ../datasets/issue-754.yaml \ + --suite ../suites/issue-754.yaml \ + --compose-file ../docker-compose.yml \ + --compose-file ../docker-compose.benefit.yml \ + --output "$(REPORT_DIR)/issue-754-benefit.json" \ + --logs-dir "$(LOGS_DIR)/issue-754-benefit" diff --git a/promql-compliance/runner/src/bin/benefit-runner.rs b/promql-compliance/runner/src/bin/benefit-runner.rs new file mode 100644 index 00000000..d9f15759 --- /dev/null +++ b/promql-compliance/runner/src/bin/benefit-runner.rs @@ -0,0 +1,5 @@ +use clap::Parser; +#[tokio::main] +async fn main() -> anyhow::Result<()> { + promql_compliance::runner::run(promql_compliance::runner::Args::parse(), true).await +}