diff --git a/.github/production-repositories.jsonc b/.github/production-repositories.jsonc new file mode 100644 index 0000000..4d21eac --- /dev/null +++ b/.github/production-repositories.jsonc @@ -0,0 +1,9 @@ +// Public GitHub repositories served by the production remote cache. +// Each key is a cache namespace. Its endpoint is +// https://voidzero-remote-cache..workers.dev/projects/ +// Each production deployment enables every listed namespace and disables every other namespace, +// overriding `pnpm operator policy`. Data in a removed namespace expires with its retention. +{ + "rolldown": "rolldown/rolldown", + "vite-plus": "voidzero-dev/vite-plus" +} diff --git a/.github/workflows/remote-cache-deploy.yml b/.github/workflows/remote-cache-deploy.yml index 991baed..c70a423 100644 --- a/.github/workflows/remote-cache-deploy.yml +++ b/.github/workflows/remote-cache-deploy.yml @@ -81,7 +81,7 @@ jobs: - uses: oxc-project/setup-node@f46a72f95efdc55273fcd042d61c84e723b2892c # v1.4.1 - name: Create or update persistent staging resources id: deploy - run: pnpm ci:deploy + run: pnpm ci:deploy staging env: CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} diff --git a/.github/workflows/remote-cache-production.yml b/.github/workflows/remote-cache-production.yml new file mode 100644 index 0000000..5e31f1d --- /dev/null +++ b/.github/workflows/remote-cache-production.yml @@ -0,0 +1,35 @@ +name: Remote cache production + +on: + workflow_dispatch: + +permissions: + contents: read + +# Finish one production deployment before starting the next. +concurrency: + group: remote-cache-production + cancel-in-progress: false + +jobs: + deploy: + name: Deploy production + runs-on: ubuntu-latest + timeout-minutes: 15 + environment: + name: production + url: ${{ steps.deploy.outputs.url }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - uses: oxc-project/setup-node@f46a72f95efdc55273fcd042d61c84e723b2892c # v1.4.1 + - run: pnpm check + - run: pnpm smoke + # The script accepts only manual runs of this repository's main branch. + - name: Deploy the production Worker, D1, and R2 + id: deploy + run: pnpm ci:deploy production + env: + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} diff --git a/README.md b/README.md index 7583698..2b7c6e4 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ The repository created by the deployment button runs the same commands. `pnpm ch For your own service and application repository, use the [self-hosting guide](docs/self-hosting.md). The commands below are the operator reference. -For continuous deployment and smoke tests against one persistent Cloudflare staging environment, follow the [deployment and e2e plan](docs/e2e-plan.md). Internal PRs and main-branch pushes share the same Worker, D1 database, and R2 bucket. The plan includes GitHub configuration, the complete test matrix, and manual verification. Closing a PR leaves staging available. +For continuous deployment and smoke tests against one persistent Cloudflare staging environment, follow the [deployment and e2e plan](docs/e2e-plan.md). Internal PRs and main-branch pushes share the same Worker, D1 database, and R2 bucket. The plan includes GitHub configuration, the complete test matrix, and manual verification. Closing a PR leaves staging available. Maintainers deploy this repository's production cache manually; see [Production deployment](docs/e2e-plan.md#production-deployment). Use a dedicated Worker, D1 database, and bucket. The operator requires `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_API_TOKEN` in its environment. The token needs account permissions for Workers Scripts, D1, and Workers R2 Storage, plus route/zone permissions if using a custom domain. Enable R2 in the account first. Credentials stay in the operator process and Wrangler; they are never stored in namespace policy or passed as command arguments. diff --git a/docs/e2e-plan.md b/docs/e2e-plan.md index 353ef14..648558e 100644 --- a/docs/e2e-plan.md +++ b/docs/e2e-plan.md @@ -169,6 +169,18 @@ Subsequent runs restore CI-owned policy switches in case a prior process stopped Staging policies belong to CI. Do not use these names for an operator-managed production cache. Monitor storage, deletion backlogs, and Cloudflare allowances. Resource budgets do not guarantee zero cost. Set `REMOTE_CACHE_DEPLOY_ENABLED=false` to stop automatic deployments; the existing staging environment remains available. +## Production deployment + +`.github/workflows/remote-cache-production.yml` deploys this repository's production cache. It runs only when a maintainer starts it from **Actions → Remote cache production → Run workflow** on `main`. Pushes and PRs never deploy production. Before starting it, check that the commit's main-branch staging run passed. + +The workflow repeats `pnpm check` and `pnpm smoke`, then runs `pnpm ci:deploy production`. `scripts/ci/production.ts` names the Worker, D1 database, and R2 bucket. `.github/production-repositories.jsonc` maps each cache namespace to a public GitHub repository. The command refuses other repositories, branches, and events, so repositories created by Deploy to Cloudflare cannot run it. Staging and production share `scripts/ci/deploy.ts`. Setup creates or reuses the named resources, applies migrations, and binds each namespace; then the Worker deploys once. Every namespace endpoint must then serve the new deployment ID. The run summary and the `production` environment list the endpoints, for example `https://voidzero-remote-cache..workers.dev/projects/rolldown`. + +To bind another repository, add `"namespace": "owner/repository"` to `.github/production-repositories.jsonc`, merge the change, and run the workflow. Only public repositories can publish. The list decides which namespaces serve reads and accept uploads: each deployment enables every listed namespace and disables every other namespace, so a `pnpm operator policy` change lasts only until the next deployment. To withdraw a repository, remove its entry and run the workflow. For an immediate stop, also run `pnpm operator policy --namespace --enabled off`. Data in a disabled namespace expires with its retention; `pnpm operator purge` deletes it sooner. Deployments never change the cache-wide `pnpm operator deployment` switches or move a namespace to another repository. To roll back, re-run an earlier successful production run. A re-run deploys its original commit. + +Production uses the same `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_API_TOKEN` secrets as staging. Secrets in the `production` environment take precedence, so a production-only token can replace them without workflow changes. Internal PR staging runs receive the repository secrets; while both environments share one token, anyone who can push a branch here can change production resources. Add required reviewers to the `production` environment to approve each deployment. + +Operator commands read `wrangler.operator.json`, which the workflow does not keep. To recreate it, check out the deployed commit and repeat `pnpm operator setup` with the Worker name from `scripts/ci/production.ts` and an entry from `.github/production-repositories.jsonc`. Setup redeploys that checkout. + ## Release and incident exercises Before a production release, record the commit, workflow URL, Cloudflare plan, region, and outcome for these controlled staging exercises: diff --git a/scripts/ci.ts b/scripts/ci.ts index f659b28..0a845cc 100644 --- a/scripts/ci.ts +++ b/scripts/ci.ts @@ -2,19 +2,24 @@ import assert from 'node:assert/strict'; import { appendFile, mkdir, writeFile } from 'node:fs/promises'; import { URL, pathToFileURL } from 'node:url'; import { encode } from 'cborg'; -import { ApiError, operatorIO, query, runOperator, type Config } from './operator.ts'; +import { + deployTarget, + reportDeployment, + runContext, + type Deployment, + type RunContext, +} from './ci/deploy.ts'; +import { productionTarget } from './ci/production.ts'; +import { ApiError, operatorIO, query, type Config } from './operator.ts'; import { retireTestData, seedManual, type Admin } from './e2e/fixtures.ts'; import { runSuite, type Report } from './e2e/suite.ts'; import { readJson } from './http.ts'; const resultsDir = new URL('../e2e-results/', import.meta.url); -interface Settings { +interface Settings extends RunContext { name: string; - repository: string; - repositoryId: string; - revision: string; - deployment: string; + subdomain: string; origin: string; writes: boolean; profile: 'free' | 'paid'; @@ -35,33 +40,18 @@ export function settingsFrom(env: Record): Settings 'Set REMOTE_CACHE_WORKERS_SUBDOMAIN to the account subdomain, without workers.dev', ); const name = `${prefix}-staging`; - const repository = env['GITHUB_REPOSITORY']; - if (!repository || !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repository)) - throw new Error('Invalid repository'); - const repositoryId = env['GITHUB_REPOSITORY_ID']; - if (!repositoryId || !/^[1-9][0-9]*$/.test(repositoryId)) - throw new Error('Invalid repository ID'); - const revision = env['REMOTE_CACHE_SOURCE_SHA']; - if (!revision || !/^[a-f0-9]{40}$/.test(revision)) - throw new Error('Use the full source commit SHA'); - const run = env['GITHUB_RUN_ID']; - const attempt = env['GITHUB_RUN_ATTEMPT']; - if (!run || !attempt || !/^\d+$/.test(run) || !/^\d+$/.test(attempt)) - throw new Error('Invalid workflow run identity'); + const context = runContext(env); const defaultBranch = env['REMOTE_CACHE_DEFAULT_BRANCH']; - const event = env['GITHUB_EVENT_NAME']; - if (!['pull_request', 'push', 'workflow_dispatch'].includes(event ?? '')) + if (!['pull_request', 'push', 'workflow_dispatch'].includes(context.event)) throw new Error('Unsupported staging deployment event'); - const onDefaultBranch = env['GITHUB_REF'] === `refs/heads/${defaultBranch}`; - if (event !== 'pull_request' && !onDefaultBranch) + const onDefaultBranch = context.ref === `refs/heads/${defaultBranch}`; + if (context.event !== 'pull_request' && !onDefaultBranch) throw new Error('Push and manual staging deployments require the default branch'); - const writes = event === 'push' && onDefaultBranch; + const writes = context.event === 'push' && onDefaultBranch; return { + ...context, name, - repository, - repositoryId, - revision, - deployment: `${revision}-${run}-${attempt}`, + subdomain, origin: `https://${name}.${subdomain}.workers.dev`, writes, profile, @@ -163,88 +153,64 @@ export function githubTokens( }; } -async function deploy(settings: Settings): Promise { - await checkAccountOrigin(settings); - await runOperator( - [ - 'setup', - '--name', - settings.name, - '--namespace', - 'e2e', - '--repo', - settings.repository, - '--origin', - settings.origin, - '--profile', - settings.profile, - '--retention-days', - '1', - '--byte-limit', - '2000000000', - '--entry-limit', - '1000', - '--association-limit', - '2000', - ], +async function deploy(settings: Settings): Promise { + const result = await deployTarget( { - ...operatorIO, - async wrangler(args) { - // Install all CI namespace policies before the single final deployment. - if (args[0] !== 'deploy') await operatorIO.wrangler(args); - }, - async writeConfig(config) { - config.vars['DEPLOYMENT_ID'] = settings.deployment; - await operatorIO.writeConfig(config); - }, - }, - ); - const config = await operatorIO.readConfig(); - checkConfig(config, settings); - const existing = await query( - operatorIO, - config, - 'SELECT scope_id, repository_id, endpoint FROM scopes', - ); - if ( - existing.some( - (scope) => - !['e2e', 'other', 'manual'].includes(String(scope['scope_id'])) || - scope['repository_id'] !== settings.repositoryId || - scope['endpoint'] !== `${settings.origin}/projects/${String(scope['scope_id'])}`, - ) - ) - throw new Error('CI resources must contain only this repository’s verification namespaces'); - for (const scope of ['other', 'manual']) - await query( - operatorIO, - config, - `INSERT INTO scopes + name: settings.name, + subdomain: settings.subdomain, + profile: settings.profile, + bindings: { e2e: settings.repository }, + setupArgs: [ + '--retention-days', + '1', + '--byte-limit', + '2000000000', + '--entry-limit', + '1000', + '--association-limit', + '2000', + ], + async prepare(config, io) { + checkConfig(config, settings); + const existing = await query( + io, + config, + 'SELECT scope_id, repository_id, endpoint FROM scopes', + ); + if ( + existing.some( + (scope) => + !['e2e', 'other', 'manual'].includes(String(scope['scope_id'])) || + scope['repository_id'] !== settings.repositoryId || + scope['endpoint'] !== `${settings.origin}/projects/${String(scope['scope_id'])}`, + ) + ) + throw new Error( + 'CI resources must contain only this repository’s verification namespaces', + ); + for (const scope of ['other', 'manual']) + await query( + io, + config, + `INSERT INTO scopes (scope_id, endpoint, repository, repository_id, repository_owner_id, branch, retention_seconds) SELECT ?, ?, repository, repository_id, repository_owner_id, branch, 86400 FROM scopes WHERE scope_id = 'e2e' ON CONFLICT(scope_id) DO NOTHING`, - [scope, `${settings.origin}/projects/${scope}`], - ); - // Recover policy changes left by an interrupted e2e run. These resources belong to CI only. - await query(operatorIO, config, 'UPDATE deployment SET enabled = 1, writes_enabled = 1'); - await query( - operatorIO, - config, - `UPDATE scopes SET enabled = 1, writes_enabled = 1, + [scope, `${settings.origin}/projects/${scope}`], + ); + // Recover policy changes left by an interrupted e2e run. These resources belong to CI only. + await query(io, config, 'UPDATE deployment SET enabled = 1, writes_enabled = 1'); + await query( + io, + config, + `UPDATE scopes SET enabled = 1, writes_enabled = 1, byte_limit = 2000000000, entry_limit = 1000, association_limit = 2000`, + ); + }, + }, + settings, ); - config.vars['NAMESPACES'] = '["e2e","other","manual"]'; - await operatorIO.writeConfig(config); - await operatorIO.wrangler(['deploy', '--config', 'wrangler.operator.json']); - const managed = (await operatorIO.api(`/r2/buckets/${settings.name}/domains/managed`)) as { - enabled: boolean; - }; - const custom = (await operatorIO.api(`/r2/buckets/${settings.name}/domains/custom`)) as { - domains: unknown[]; - }; - assert.equal(managed.enabled, false, 'R2 must remain private'); - assert.deepEqual(custom.domains, [], 'R2 must have no public custom domain'); - const fixture = await seedManual(cloudflareAdmin(config), settings.deployment); + const fixture = await seedManual(cloudflareAdmin(result.config), settings.deployment); await mkdir(resultsDir, { recursive: true }); await writeFile( new URL('manual-fetch.cbor', resultsDir), @@ -255,7 +221,7 @@ async function deploy(settings: Settings): Promise { JSON.stringify( { deployment: settings.deployment, - source_sha: settings.revision, + source_sha: settings.sha, endpoint: `${settings.origin}/projects/manual`, blob_url: `${settings.origin}/projects/manual/blob/${fixture.blobId}`, value_utf8: Buffer.from(fixture.value).toString(), @@ -270,6 +236,7 @@ async function deploy(settings: Settings): Promise { process.env['GITHUB_OUTPUT'], `endpoint=${settings.origin}/projects/manual\ndeployment=${settings.deployment}\n`, ); + return result; } async function verify(settings: Settings): Promise { @@ -309,11 +276,14 @@ async function verify(settings: Settings): Promise { if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { try { - const settings = settingsFrom(process.env); - const command = process.argv[2]; - if (command === 'deploy') await deploy(settings); - else if (command === 'test') await verify(settings); - else throw new Error('Use deploy or test'); + const [command, target] = process.argv.slice(2); + if (command === 'deploy' && target === 'staging') + await reportDeployment(await deploy(settingsFrom(process.env))); + else if (command === 'deploy' && target === 'production') { + const context = runContext(process.env); + await reportDeployment(await deployTarget(productionTarget(context), context)); + } else if (command === 'test') await verify(settingsFrom(process.env)); + else throw new Error('Use deploy staging, deploy production, or test'); } catch (error) { console.error(error instanceof Error ? error.message : 'Cloudflare CI failed'); process.exitCode = 1; diff --git a/scripts/ci/deploy.ts b/scripts/ci/deploy.ts new file mode 100644 index 0000000..004a20a --- /dev/null +++ b/scripts/ci/deploy.ts @@ -0,0 +1,142 @@ +import assert from 'node:assert/strict'; +import { appendFile } from 'node:fs/promises'; +import { setTimeout } from 'node:timers/promises'; +import { namespaceEnabled, waitForDeployment } from '../deploy.ts'; +import { operatorIO, query, runOperator, type Config, type OperatorIO } from '../operator.ts'; + +export interface RunContext { + repository: string; + repositoryId: string; + sha: string; + deployment: string; + event: string; + ref: string; +} + +// Each target applies its own event and branch rules to this identity. +export function runContext(env: Record): RunContext { + const repository = env['GITHUB_REPOSITORY']; + if (!repository || !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repository)) + throw new Error('Invalid repository'); + const repositoryId = env['GITHUB_REPOSITORY_ID']; + if (!repositoryId || !/^[1-9][0-9]*$/.test(repositoryId)) + throw new Error('Invalid repository ID'); + const sha = env['REMOTE_CACHE_SOURCE_SHA'] || env['GITHUB_SHA']; + if (!sha || !/^[a-f0-9]{40}$/.test(sha)) throw new Error('Use the full source commit SHA'); + const run = env['GITHUB_RUN_ID']; + const attempt = env['GITHUB_RUN_ATTEMPT']; + if (!run || !attempt || !/^\d+$/.test(run) || !/^\d+$/.test(attempt)) + throw new Error('Invalid workflow run identity'); + return { + repository, + repositoryId, + sha, + deployment: `${sha}-${run}-${attempt}`, + event: env['GITHUB_EVENT_NAME'] ?? '', + ref: env['GITHUB_REF'] ?? '', + }; +} + +export interface Target { + name: string; + // Expected account subdomain. Defaults to the authenticated account's subdomain. + subdomain?: string; + profile: 'free' | 'paid'; + // Namespace to public GitHub owner/repository. + bindings: Readonly>; + setupArgs?: string[]; + // Runs after every namespace is set up, before the Worker deployment. + prepare?(config: Config, io: OperatorIO): Promise; +} + +export interface Deployment { + config: Config; + deployment: string; + endpoints: { namespace: string; endpoint: string; enabled: boolean }[]; +} + +export async function deployTarget( + target: Target, + context: Pick, + io: OperatorIO = operatorIO, + request: typeof fetch = fetch, + wait: (ms: number) => Promise = setTimeout, +): Promise { + const namespaces = Object.keys(target.bindings); + if (!namespaces.length) throw new Error('Bind at least one namespace'); + const account = (await io.api('/workers/subdomain')) as { subdomain?: unknown }; + if (typeof account.subdomain !== 'string' || !/^[a-z0-9][a-z0-9-]{0,62}$/.test(account.subdomain)) + throw new Error('Create a workers.dev subdomain in your Cloudflare account before deploying'); + if (target.subdomain !== undefined) + assert.equal( + account.subdomain, + target.subdomain, + 'The configured Workers subdomain must belong to the authenticated Cloudflare account', + ); + const origin = `https://${target.name}.${account.subdomain}.workers.dev`; + const setupIO: OperatorIO = { + ...io, + async wrangler(args) { + // Install every namespace policy before the single final deployment. + if (args[0] !== 'deploy') await io.wrangler(args); + }, + async writeConfig(config) { + config.vars['DEPLOYMENT_ID'] = context.deployment; + await io.writeConfig(config); + }, + }; + for (const namespace of namespaces) + await runOperator( + [ + 'setup', + '--name', + target.name, + '--namespace', + namespace, + '--repo', + target.bindings[namespace]!, + '--origin', + origin, + '--profile', + target.profile, + ...(target.setupArgs ?? []), + ], + setupIO, + ); + const config = await io.readConfig(); + await target.prepare?.(config, io); + const scopes = await query(io, config, 'SELECT scope_id FROM scopes'); + config.vars['NAMESPACES'] = JSON.stringify(scopes.map((scope) => String(scope['scope_id']))); + await io.writeConfig(config); + await io.wrangler(['deploy', '--config', 'wrangler.operator.json']); + const bucket = config.r2_buckets[0]!.bucket_name; + const managed = (await io.api(`/r2/buckets/${bucket}/domains/managed`)) as { enabled: boolean }; + const custom = (await io.api(`/r2/buckets/${bucket}/domains/custom`)) as { domains: unknown[] }; + assert.equal(managed.enabled, false, 'R2 must remain private'); + assert.deepEqual(custom.domains, [], 'R2 must have no public custom domain'); + const endpoints: Deployment['endpoints'] = []; + for (const namespace of namespaces) { + const endpoint = `${origin}/projects/${namespace}`; + const enabled = await namespaceEnabled(io, config, namespace); + await waitForDeployment(endpoint, context.deployment, enabled, request, wait); + endpoints.push({ namespace, endpoint, enabled }); + } + return { config, deployment: context.deployment, endpoints }; +} + +export async function reportDeployment( + result: Deployment, + env: Record = process.env, +): Promise { + if (env['GITHUB_OUTPUT']) + await appendFile(env['GITHUB_OUTPUT'], `url=${result.endpoints[0]!.endpoint}\n`); + if (env['GITHUB_STEP_SUMMARY']) + await appendFile( + env['GITHUB_STEP_SUMMARY'], + `### Remote cache deployment\n\nWorker: \`${result.config.name}\`. Deployment: \`${result.deployment}\`\n\n` + + result.endpoints + .map(({ endpoint, enabled }) => `- ${endpoint}${enabled ? '' : ' (disabled)'}`) + .join('\n') + + '\n', + ); +} diff --git a/scripts/ci/production.ts b/scripts/ci/production.ts new file mode 100644 index 0000000..b65447c --- /dev/null +++ b/scripts/ci/production.ts @@ -0,0 +1,84 @@ +import { readFileSync } from 'node:fs'; +import { URL } from 'node:url'; +import { parse, type ParseError } from 'jsonc-parser'; +import { query } from '../operator.ts'; +import type { RunContext, Target } from './deploy.ts'; + +interface ProductionConfig { + source: string; + ref: string; + name: string; + profile: Target['profile']; +} + +export const production: ProductionConfig = { + source: 'voidzero-dev/vite-plus-remote-cache-cloudflare', + ref: 'refs/heads/main', + name: 'voidzero-remote-cache', + profile: 'free', +}; + +// Add or remove production repositories in this file. +const repositoriesFile = new URL('../../.github/production-repositories.jsonc', import.meta.url); + +export function parseRepositories(text: string): Record { + const errors: ParseError[] = []; + const value: unknown = parse(text, errors, { allowTrailingComma: true }); + if (errors.length || !value || typeof value !== 'object' || Array.isArray(value)) + throw new Error('Production repositories must be a JSONC object of namespace to owner/repo'); + const entries = Object.entries(value); + if (!entries.length || entries.length > 100) + throw new Error('List between 1 and 100 production repositories'); + for (const [namespace, repository] of entries) + if ( + !/^[a-z0-9][a-z0-9-]{0,62}$/.test(namespace) || + typeof repository !== 'string' || + !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repository) + ) + throw new Error(`Invalid production repository entry: ${namespace}`); + return Object.fromEntries(entries) as Record; +} + +export function productionTarget( + context: Pick, + config: ProductionConfig = production, + repositories: string = readFileSync(repositoriesFile, 'utf8'), +): Target { + // Repositories created by Deploy to Cloudflare copy this workflow; they must not deploy it. + if (context.repository !== config.source) + throw new Error(`Production deploys only from ${config.source}`); + if (context.event !== 'workflow_dispatch') + throw new Error('Start production deployments manually'); + if (context.ref !== config.ref) throw new Error(`Production deploys only from ${config.ref}`); + if (/-(ci|staging)$/.test(config.name)) + throw new Error('Production cannot use CI staging resources'); + const bindings = parseRepositories(repositories); + const listed = JSON.stringify(Object.keys(bindings)); + return { + name: config.name, + profile: config.profile, + bindings, + // The list decides which namespaces serve reads and accept uploads, overriding + // `pnpm operator policy`. Only changed rows are updated: the changed_policy trigger bumps + // policy_version on any switch update, which stops in-flight uploads from publishing. + async prepare(deployed, io) { + await query( + io, + deployed, + `UPDATE scopes SET enabled = 1, writes_enabled = 1 + WHERE scope_id IN (SELECT value FROM json_each(?)) AND (enabled = 0 OR writes_enabled = 0)`, + [listed], + ); + const disabled = await query( + io, + deployed, + `UPDATE scopes SET enabled = 0, writes_enabled = 0 + WHERE scope_id NOT IN (SELECT value FROM json_each(?)) AND (enabled = 1 OR writes_enabled = 1) + RETURNING scope_id`, + [listed], + ); + for (const scope of disabled) + io.print(`Disabled namespace ${String(scope['scope_id'])}: it is not in the repository list`); + }, + }; +} diff --git a/scripts/deploy.ts b/scripts/deploy.ts index e8a4f56..0406aad 100644 --- a/scripts/deploy.ts +++ b/scripts/deploy.ts @@ -44,6 +44,40 @@ export async function checkDeployment( } } +export async function waitForDeployment( + endpoint: string, + deployment: string, + enabled: boolean, + request: typeof fetch = fetch, + wait: (ms: number) => Promise = setTimeout, +): Promise { + // New workers.dev routes and revisions can take a short time to reach the edge. + for (let attempt = 0; ; attempt++) { + try { + await checkDeployment(endpoint, deployment, enabled, request); + return; + } catch (error) { + if (attempt === 9) throw error; + await wait(3000); + } + } +} + +export async function namespaceEnabled( + io: OperatorIO, + config: Config, + namespace: string, +): Promise { + const policies = await query( + io, + config, + 'SELECT scopes.enabled AS scope_enabled, deployment.enabled AS deployment_enabled FROM scopes CROSS JOIN deployment WHERE scope_id = ?', + [namespace], + ); + if (policies.length !== 1) throw new Error('The deployed namespace has no policy'); + return policies[0]!['scope_enabled'] === 1 && policies[0]!['deployment_enabled'] === 1; +} + export async function deploy( config: Config, io: OperatorIO = operatorIO, @@ -103,25 +137,9 @@ export async function deploy( { database, bucket }, ); const deployed = await io.readConfig(); - const policies = await query( - io, - deployed, - 'SELECT scopes.enabled AS scope_enabled, deployment.enabled AS deployment_enabled FROM scopes CROSS JOIN deployment WHERE scope_id = ?', - [namespace], - ); - if (policies.length !== 1) throw new Error('The deployed namespace has no policy'); - const enabled = policies[0]!['scope_enabled'] === 1 && policies[0]!['deployment_enabled'] === 1; + const enabled = await namespaceEnabled(io, deployed, namespace); const endpoint = `${origin}/projects/${namespace}`; - // New workers.dev routes and revisions can take a short time to reach the edge. - for (let attempt = 0; ; attempt++) { - try { - await checkDeployment(endpoint, deployment, enabled, request); - break; - } catch (error) { - if (attempt === 9) throw error; - await wait(3000); - } - } + await waitForDeployment(endpoint, deployment, enabled, request, wait); io.print(`Deployment checks passed. Cache endpoint: ${endpoint}`); if (!enabled) io.print('This namespace remains disabled. Deployment did not change its access policy.'); diff --git a/test/ci.test.ts b/test/ci.test.ts new file mode 100644 index 0000000..c2cd707 --- /dev/null +++ b/test/ci.test.ts @@ -0,0 +1,260 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { deployTarget, runContext, type Target } from '../scripts/ci/deploy.ts'; +import { parseRepositories, production, productionTarget } from '../scripts/ci/production.ts'; +import { readTemplate, type Config, type OperatorIO } from '../scripts/operator.ts'; +import { harness } from './helpers.ts'; + +const sha = 'a'.repeat(40); +const deployment = `${sha}-42-1`; +const database = '12345678-1234-1234-1234-123456789abc'; +const dispatch = { + repository: production.source, + event: 'workflow_dispatch', + ref: 'refs/heads/main', +}; + +void test('CI run context identifies the source commit and workflow attempt', () => { + const env = { + GITHUB_REPOSITORY: 'owner/repo', + GITHUB_REPOSITORY_ID: '123', + GITHUB_SHA: sha, + GITHUB_RUN_ID: '42', + GITHUB_RUN_ATTEMPT: '1', + GITHUB_EVENT_NAME: 'workflow_dispatch', + GITHUB_REF: 'refs/heads/main', + }; + assert.deepEqual(runContext(env), { + repository: 'owner/repo', + repositoryId: '123', + sha, + deployment, + event: 'workflow_dispatch', + ref: 'refs/heads/main', + }); + const head = 'b'.repeat(40); + assert.equal(runContext({ ...env, REMOTE_CACHE_SOURCE_SHA: head }).sha, head); + for (const override of [ + { GITHUB_REPOSITORY: 'owner' }, + { GITHUB_REPOSITORY_ID: '0' }, + { GITHUB_SHA: 'abc' }, + { GITHUB_RUN_ATTEMPT: '' }, + ]) + assert.throws(() => runContext({ ...env, ...override })); +}); + +void test('CI deployments set up every namespace before one Worker deployment and preserve withdrawals', async () => { + const h = await harness({ + initializeDatabase: false, + namespaces: ['one', 'two'], + deploymentId: deployment, + }); + const repositories: Record = { 'acme/one': 1, 'acme/two': 2 }; + let config: Config | undefined; + let migrated = false; + const log: string[] = []; + const waits: number[] = []; + const io: OperatorIO = { + async api(path, method, body) { + if (path === '/workers/subdomain') return { subdomain: 'team' }; + if (path === '/d1/database?name=prod-cache&per_page=100') + return [{ name: 'prod-cache', uuid: database }]; + if (path === `/d1/database/${database}/query`) { + const { sql, params } = body as { sql: string; params: (string | number | null)[] }; + return [ + await h.db + .prepare(sql) + .bind(...params) + .all(), + ]; + } + if (path === '/r2/buckets/prod-cache') return { storage_class: 'Standard' }; + if (path === '/r2/buckets/prod-cache/domains/custom') return { domains: [] }; + if (path === '/r2/buckets/prod-cache/domains/managed') + return method === 'PUT' ? {} : { enabled: false }; + throw new Error(`Unexpected API request: ${path}`); + }, + async github(repository) { + return { + id: repositories[repository], + full_name: repository, + owner: { id: 456 }, + private: false, + visibility: 'public', + default_branch: 'main', + }; + }, + async wrangler(args) { + log.push(args.slice(0, 2).join(' ')); + if (args[1] === 'migrations' && !migrated) { + await h.migrate(); + migrated = true; + } + }, + async readConfig() { + return structuredClone(config!); + }, + async writeConfig(value) { + config = structuredClone(value); + }, + async lifecycle() {}, + print() {}, + }; + const request: typeof fetch = async (input, init) => { + const req = new Request(input, init); + const response = await h.mf.dispatchFetch(req.url, { + method: req.method, + headers: Object.fromEntries(req.headers), + body: await req.arrayBuffer(), + }); + return new Response(await response.arrayBuffer(), { + status: response.status, + headers: Object.fromEntries(response.headers), + }); + }; + const target: Target = { + name: 'prod-cache', + profile: 'free', + bindings: { one: 'acme/one', two: 'acme/two' }, + async prepare() { + log.push('prepare'); + }, + }; + const run = (overrides: Partial = {}) => + deployTarget({ ...target, ...overrides }, { deployment }, io, request, async (ms) => { + waits.push(ms); + }); + const deploys = () => log.filter((entry) => entry === 'prepare' || entry.startsWith('deploy')); + try { + const result = await run(); + assert.deepEqual(deploys(), ['prepare', 'deploy --config']); + assert.deepEqual(waits, []); + assert.equal(config!.vars['DEPLOYMENT_ID'], deployment); + assert.deepEqual(JSON.parse(config!.vars['NAMESPACES']!).sort(), ['one', 'two']); + assert.deepEqual(result.endpoints, [ + { + namespace: 'one', + endpoint: 'https://prod-cache.team.workers.dev/projects/one', + enabled: true, + }, + { + namespace: 'two', + endpoint: 'https://prod-cache.team.workers.dev/projects/two', + enabled: true, + }, + ]); + const scopes = await h.db + .prepare('SELECT scope_id, repository_id FROM scopes ORDER BY scope_id') + .all(); + assert.deepEqual(scopes.results, [ + { scope_id: 'one', repository_id: '1' }, + { scope_id: 'two', repository_id: '2' }, + ]); + + // A new runner has no wrangler.operator.json, and redeployment must not re-enable a namespace. + await h.db.prepare("UPDATE scopes SET enabled = 0 WHERE scope_id = 'two'").run(); + config = undefined; + log.length = 0; + const retry = await run(); + assert.deepEqual(deploys(), ['prepare', 'deploy --config']); + assert.equal(retry.endpoints[1]!.enabled, false); + + log.length = 0; + repositories['acme/two'] = 3; + await assert.rejects(run(), /different repository/); + assert.deepEqual(deploys(), []); + + log.length = 0; + await assert.rejects(run({ subdomain: 'other' }), /Workers subdomain/); + assert.deepEqual(log, []); + } finally { + await h.close(); + } +}); + +void test('production deploys only manual default-branch runs from this repository', () => { + const context = dispatch; + // Also validates the committed repository list. + const target = productionTarget(context); + assert.equal(target.name, production.name); + assert.equal(target.profile, production.profile); + assert.ok(Object.keys(target.bindings).length > 0); + assert.throws(() => productionTarget({ ...context, event: 'push' }), /manually/); + assert.throws(() => productionTarget({ ...context, ref: 'refs/heads/feature' }), /main/); + assert.throws(() => productionTarget({ ...context, repository: 'someone/copy' }), /voidzero-dev/); + for (const name of ['vp-cache-ci', 'vp-cache-ci-staging']) + assert.throws(() => productionTarget(context, { ...production, name }), /staging/); +}); + +void test('production repository lists map namespaces to public owner/repo names', () => { + assert.deepEqual( + parseRepositories('// comment\n{ "one": "acme/one", "two-x": "Acme/two.js", }'), + { one: 'acme/one', 'two-x': 'Acme/two.js' }, + ); + for (const text of [ + '', + '{}', + '["acme/one"]', + '{ "One": "acme/one" }', + '{ "-one": "acme/one" }', + '{ "one": "https://github.com/acme/one" }', + '{ "one": 1 }', + '{ "one": "acme/one" ', + ]) + assert.throws(() => parseRepositories(text), /production repositor/i); +}); + +void test('production deployments enable listed namespaces and disable all others', async () => { + const h = await harness(); + const config = await readTemplate(); + config.d1_databases[0]!.database_id = database; + const printed: string[] = []; + const unexpected = async () => { + throw new Error('Unexpected operator call'); + }; + const io: OperatorIO = { + async api(path, _method, body) { + assert.equal(path, `/d1/database/${database}/query`); + const { sql, params } = body as { sql: string; params: (string | number | null)[] }; + return [ + await h.db + .prepare(sql) + .bind(...params) + .all(), + ]; + }, + github: unexpected, + wrangler: unexpected, + readConfig: unexpected, + writeConfig: unexpected, + lifecycle: unexpected, + print: (message) => printed.push(message), + }; + const policies = async () => + ( + await h.db + .prepare('SELECT scope_id, enabled, writes_enabled, policy_version FROM scopes ORDER BY scope_id') + .all() + ).results; + const target = productionTarget(dispatch, production, '{ "test": "owner/repo" }'); + const expected = [ + { scope_id: 'other', enabled: 0, writes_enabled: 0, policy_version: 2 }, + { scope_id: 'test', enabled: 1, writes_enabled: 1, policy_version: 3 }, + ]; + try { + // A manual upload pause on a listed namespace lasts only until the next deployment. + // The pause and its reversal each bump policy_version. + await h.db.prepare("UPDATE scopes SET writes_enabled = 0 WHERE scope_id = 'test'").run(); + await target.prepare!(config, io); + assert.deepEqual(await policies(), expected); + assert.deepEqual(printed, ['Disabled namespace other: it is not in the repository list']); + + // A deployment that changes no switch leaves in-flight uploads alone. + printed.length = 0; + await target.prepare!(config, io); + assert.deepEqual(await policies(), expected); + assert.deepEqual(printed, []); + } finally { + await h.close(); + } +}); diff --git a/test/index.test.ts b/test/index.test.ts index cf2cbdf..29f12a9 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -4,3 +4,4 @@ import './operator.test.ts'; import './deploy.test.ts'; import './failures.test.ts'; import './deployed.test.ts'; +import './ci.test.ts';