From 64cc93796d3646479e63a57121f19f35f417b668 Mon Sep 17 00:00:00 2001 From: Wesley Wigham Date: Fri, 18 Sep 2026 11:46:06 -0700 Subject: [PATCH 1/6] Make all the codegen we do incremental --- Herebyfile.mjs | 75 ++- packages/typescript/package.json | 2 +- packages/typescript/scripts/generate.ts | 9 + packages/typescript/scripts/generateSync.ts | 44 +- tools/scripts/gen/generateAPI.mts | 38 ++ tools/scripts/gen/generateBundled.mts | 40 ++ tools/scripts/gen/generateDiagnostics.mts | 55 ++ tools/scripts/gen/generateMoq.mts | 47 ++ tools/scripts/gen/generateStringer.mts | 49 ++ tools/scripts/gen/generatedFile.mts | 78 +++ tools/scripts/gen/generatedFile.test.mts | 488 ++++++++++++++++++ tools/scripts/gen/tsconfig.json | 11 + tools/scripts/gen/utils.mts | 56 ++ tools/scripts/generatedFile.mts | 77 +++ tools/scripts/generatedFile.test.mts | 488 ++++++++++++++++++ tools/scripts/tsc/generate-encoder.ts | 52 +- tools/scripts/tsc/generate-go-ast.ts | 35 +- tools/scripts/tsc/generate-ts-ast.ts | 34 +- tools/scripts/tsc/generate.ts | 11 +- tools/scripts/tsc/schema.ts | 2 +- tools/scripts/tsc/tsconfig.json | 15 +- tsc/internal/ast/kind_generated.go | 3 +- tsc/internal/bundled/bundled.go | 2 +- tsc/internal/checker/types.go | 3 +- tsc/internal/core/compileroptions.go | 5 +- tsc/internal/core/languagevariant.go | 3 +- tsc/internal/core/scriptkind.go | 3 +- tsc/internal/core/tristate.go | 3 +- tsc/internal/diagnostics/diagnostics.go | 5 +- tsc/internal/ls/autoimport/export.go | 3 +- tsc/internal/project/project.go | 3 +- .../_scripts/generate-unicode-data.mts | 37 +- tsc/internal/stringutil/generate.go | 1 - .../projecttestutil/projecttestutil.go | 6 +- tsc/internal/vfs/vfs.go | 3 +- tsc/internal/vfs/vfsmatch/vfsmatch.go | 3 +- 36 files changed, 1632 insertions(+), 157 deletions(-) create mode 100644 packages/typescript/scripts/generate.ts create mode 100644 tools/scripts/gen/generateAPI.mts create mode 100644 tools/scripts/gen/generateBundled.mts create mode 100644 tools/scripts/gen/generateDiagnostics.mts create mode 100644 tools/scripts/gen/generateMoq.mts create mode 100644 tools/scripts/gen/generateStringer.mts create mode 100644 tools/scripts/gen/generatedFile.mts create mode 100644 tools/scripts/gen/generatedFile.test.mts create mode 100644 tools/scripts/gen/tsconfig.json create mode 100644 tools/scripts/gen/utils.mts create mode 100644 tools/scripts/generatedFile.mts create mode 100644 tools/scripts/generatedFile.test.mts diff --git a/Herebyfile.mjs b/Herebyfile.mjs index dc15fc6fca1c0..391605d9b8154 100644 --- a/Herebyfile.mjs +++ b/Herebyfile.mjs @@ -105,6 +105,7 @@ const { values: rawOptions } = parseArgs({ options: { tests: { type: "string", short: "t" }, fix: { type: "boolean" }, + force: { type: "boolean", default: parseEnvBoolean("FORCE") }, api: { type: "boolean" }, all: { type: "boolean" }, debug: { type: "boolean" }, @@ -369,7 +370,10 @@ export const cleanBuilt = task({ }); async function runGenerate() { - return await run("go", ["generate", "-v", "./..."], { cwd: "./tsc" }); + return await run("go", ["generate", "-v", "./..."], { + cwd: "./tsc", + env: { TSGO_HEREBY_FORCE: options.force ? "1" : "0" }, + }); } export const generate = task({ @@ -775,9 +779,10 @@ const enumValuesGeneratedGoPath = "tsc/internal/api/enum_values_generated.go"; * because it already imports (nearly) every package enums are sourced from. * * @param {GeneratedEnum[]} generatedEnums + * @param {import("./tools/scripts/gen/generatedFile.mts").GeneratedFile} generatedGoFile * @returns {Promise>>} enum def name -> (memberName -> Go value) */ -async function computeGoGroundTruth(generatedEnums) { +async function computeGoGroundTruth(generatedEnums, generatedGoFile) { /** @type {Map} */ const packagesByDir = new Map(); /** @@ -849,7 +854,7 @@ func toInt32[T ~int8 | ~int16 | ~int32 | ~int | ~uint8 | ~uint16 | ~uint32](v T) `; - fs.writeFileSync(enumValuesGeneratedGoPath, goSource); + generatedGoFile.write(goSource); await run("dprint", ["fmt", enumValuesGeneratedGoPath]); const { stdout } = await runOutput("go", ["run", enumValuesGeneratedGoPath]); @@ -873,6 +878,26 @@ async function evaluateEnumMembers(enumSource, enumName) { } async function runGenerateEnums() { + const { GeneratedFile } = await import("./tools/scripts/gen/generatedFile.mts"); + const inputs = [ + __filename, + ...fs.globSync(["go.work", "go.work.sum", "{tsc,tools}/go.{mod,sum}"]), + ]; + const enumFiles = enumDefs.map(def => { + const camelName = def.name.charAt(0).toLowerCase() + def.name.slice(1); + return { + def, + camelName, + typeFile: new GeneratedFile(path.join(def.outDir, `${camelName}.enum.ts`), [...inputs, def.goFile]), + runtimeFile: new GeneratedFile(path.join(def.outDir, `${camelName}.ts`), [...inputs, def.goFile]), + }; + }); + const generatedGoFile = new GeneratedFile(enumValuesGeneratedGoPath, [...inputs, ...enumDefs.map(def => def.goFile)]); + const generatedFiles = [generatedGoFile, ...enumFiles.flatMap(({ typeFile, runtimeFile }) => [typeFile, runtimeFile])]; + if (generatedFiles.every(file => file.isCurrent(!!options.force))) { + console.log("Enums are up to date."); + return; + } const ts = /** @type {typeof import("typescript")} */ (await import("typescript")); /** @@ -903,34 +928,29 @@ async function runGenerateEnums() { console.log("Generating enums from Go source..."); /** @type {Array} */ const generatedEnums = []; - for (const def of enumDefs) { + for (const { def, camelName, typeFile, runtimeFile } of enumFiles) { const members = parseGoEnum(def); - const camelName = def.name.charAt(0).toLowerCase() + def.name.slice(1); - - fs.mkdirSync(def.outDir, { recursive: true }); // Generate .enum.ts (TypeScript enum — used for types) const enumTS = renderEnumTS(def, members); - const enumPath = path.join(def.outDir, `${camelName}.enum.ts`); - fs.writeFileSync(enumPath, enumTS); + typeFile.write(enumTS); // Generate .ts (IIFE — used at runtime) const enumJsCode = transpile(enumTS, def.name); const iifeSource = convertEnumToTs(enumJsCode, def.name); - const iifePath = path.join(def.outDir, `${camelName}.ts`); - fs.writeFileSync(iifePath, iifeSource); + runtimeFile.write(iifeSource); generatedEnums.push({ code: enumJsCode, def, members, - fileNames: [enumPath, iifePath], + fileNames: [typeFile.fileName, runtimeFile.fileName], }); console.log(` ${def.name}: ${members.length} members → ${camelName}.enum.ts, ${camelName}.ts`); } console.log("Getting values from go"); - const goValuesByEnum = await computeGoGroundTruth(generatedEnums); + const goValuesByEnum = await computeGoGroundTruth(generatedEnums, generatedGoFile); /** @type {string[]} */ const mismatches = []; for (const { def, members, code } of generatedEnums) { @@ -958,29 +978,33 @@ async function runGenerateEnums() { console.log("All generated values match Go."); await run("dprint", ["fmt", ...generatedEnums.flatMap(e => e.fileNames)]); + for (const file of generatedFiles) file.markCurrent(); console.log("Done."); } export const generateEnums = task({ name: "generate:enums", - description: "Generates TypeScript enum files from Go source.", + description: "Generates TypeScript enum files from Go source. Pass --force to regenerate unchanged files.", run: runGenerateEnums, }); export const generateAST = task({ name: "generate:ast", - description: "Generates AST and encoder files from ast.json.", - run: () => run("node", ["./tools/scripts/tsc/generate.ts"]), + description: "Generates AST and encoder files from ast.json. Pass --force to regenerate unchanged files.", + run: async () => { + const { default: generate } = await import("./tools/scripts/tsc/generate.ts"); + generate(!!options.force); + }, }); async function runGenerateAPI() { - await run("go", ["-C", "./tools", "run", "./gen-proto", "../tsc/internal/api/proto.go", "../packages/typescript/src/api/proto.generated.ts"]); - await run("npx", ["dprint", "fmt", "packages/typescript/src/api/proto.generated.ts"]); + const { default: generate } = await import("./tools/scripts/gen/generateAPI.mts"); + await generate(!!options.force); } export const generateAPI = task({ name: "generate:api", - description: "Generates API files from internal/api/proto.go and internal/api/session.go.", + description: "Generates API files from internal/api/proto.go and internal/api/session.go. Pass --force to regenerate unchanged files.", run: runGenerateAPI, }); @@ -1235,6 +1259,12 @@ export const testTools = task({ run: runTestTools, }); +export const testCodegen = task({ + name: "test:codegen", + description: "Runs opt-in incremental codegen tests; excluded from validate and test:all. Because this runs asserts on build codegen, it takes awhile and is somewhat redundant.", + run: () => run("node", ["--test", "./tools/scripts/gen/*.test.mts"]), +}); + export const buildAPI = task({ name: "build:api", description: "Builds @typescript/typescript JS API.", @@ -1244,7 +1274,8 @@ export const buildAPI = task({ }); async function runBuildAPITests() { - await run("npm", ["run", "-w", "@typescript/typescript", "generate:sync"]); + const { generateSync } = await import("./packages/typescript/scripts/generateSync.ts"); + generateSync(!!options.force); await run("npm", ["run", "-w", "@typescript/typescript", "build:test"]); } @@ -1264,7 +1295,7 @@ export const testAPI = task({ export const testAll = task({ name: "test:all", - description: "Runs ALL tests in the repo, including benchmarks, tools, and the API tests.", + description: "Runs compiler, extension, benchmark, tools, and API tests. Codegen tests are opt-in via test:codegen.", dependencies: [tsgo, buildAPITests], run: async () => { // Prevent interleaving by running these directly instead of in parallel. @@ -1473,6 +1504,7 @@ export const checkHerebyfile = task({ "./node_modules/typescript/bin/tsc", "--noEmit", "--allowJs", + "--allowImportingTsExtensions", "--checkJs", "--target", "es2022", @@ -1527,6 +1559,7 @@ export const checkVsceVersion = task({ }); const scriptTsconfigs = [ + "./tools/scripts/gen/tsconfig.json", "./tools/scripts/tsc/tsconfig.json", "./tsc/internal/lsp/lsproto/_generate/tsconfig.json", ]; diff --git a/packages/typescript/package.json b/packages/typescript/package.json index 0987c2af1f950..5e26fd27a4524 100644 --- a/packages/typescript/package.json +++ b/packages/typescript/package.json @@ -97,7 +97,7 @@ }, "scripts": { "node": "node --conditions @typescript/source", - "generate": "npm run generate:ast && npm run generate:encoder && npm run generate:sync", + "generate": "npm run node -- scripts/generate.ts", "generate:ast": "node ../../tools/scripts/tsc/generate-ts-ast.ts", "generate:encoder": "npm run node -- ../../tools/scripts/tsc/generate-encoder.ts", "generate:sync": "npm run node -- scripts/generateSync.ts", diff --git a/packages/typescript/scripts/generate.ts b/packages/typescript/scripts/generate.ts new file mode 100644 index 0000000000000..19e9ba79521ef --- /dev/null +++ b/packages/typescript/scripts/generate.ts @@ -0,0 +1,9 @@ +import { parseGeneratorArgs } from "../../../tools/scripts/gen/utils.mts"; +import generateEncoder from "../../../tools/scripts/tsc/generate-encoder.ts"; +import generateAST from "../../../tools/scripts/tsc/generate-ts-ast.ts"; +import { generateSync } from "./generateSync.ts"; + +const { force } = parseGeneratorArgs({}); +generateAST(force); +generateEncoder(force); +generateSync(force); diff --git a/packages/typescript/scripts/generateSync.ts b/packages/typescript/scripts/generateSync.ts index e88072dbcd845..d236ea091ca78 100755 --- a/packages/typescript/scripts/generateSync.ts +++ b/packages/typescript/scripts/generateSync.ts @@ -21,18 +21,17 @@ * node generateSync.ts */ +import { readFileSync } from "node:fs"; import { - mkdirSync, - readFileSync, - writeFileSync, -} from "node:fs"; -import { - dirname, join, relative, } from "node:path"; -import { xSync } from "tinyexec"; import ts from "typescript"; +import { GeneratedFile } from "../../../tools/scripts/gen/generatedFile.mts"; +import { + formatFilesSync, + parseGeneratorArgs, +} from "../../../tools/scripts/gen/utils.mts"; function generatedHeader(asyncSourceRelPath: string): string { return [ @@ -59,8 +58,11 @@ function generateSyncFile( srcPath: string, destPath: string, transform: SourceTransform, + force: boolean, variant: SyncVariant = "sync", -): string { +): GeneratedFile | undefined { + const generated = new GeneratedFile(destPath, [import.meta.filename, srcPath]); + if (generated.isCurrent(force)) return; const source = readFileSync(srcPath, "utf-8"); // Normalize line endings to LF @@ -77,12 +79,11 @@ function generateSyncFile( const srcRelPath = relative(ROOT, srcPath).replaceAll("\\", "/"); result = generatedHeader(srcRelPath) + result; - mkdirSync(dirname(destPath), { recursive: true }); - writeFileSync(destPath, result); + generated.write(result); const label = relative(ROOT, srcPath).replaceAll("\\", "/"); const destLabel = relative(ROOT, destPath).replaceAll("\\", "/"); console.log(` ${label} → ${destLabel}`); - return destPath; + return generated; } // ── Directive processing ───────────────────────────────────────── @@ -626,15 +627,11 @@ function getIndent(source: string, position: number): string { // ── Formatting ─────────────────────────────────────────────────── -function formatFiles(paths: string[]): void { - xSync("dprint", ["fmt", ...paths], { throwOnError: true }); -} - // ── Main ───────────────────────────────────────────────────────── -export function generateSync(): void { +export function generateSync(force = false): void { console.log("Generating sync API from async source..."); - const generatedFiles: string[] = []; + const generatedFiles: (GeneratedFile | undefined)[] = []; // Source files for (const relPath of ["types.ts", "api.ts"]) { @@ -655,6 +652,7 @@ export function generateSync(): void { "", transformAsyncSource(source, fileName, true), ].join("\n"), + force, )); } @@ -664,6 +662,7 @@ export function generateSync(): void { join(TEST, "async", relPath), join(TEST, "sync", relPath), (source, fileName) => transformAsyncSource(source, fileName, false), + force, )); } @@ -671,14 +670,21 @@ export function generateSync(): void { join(TEST, "async", "api.bench.ts"), join(TEST, "generators", "api.bench.ts"), (source, fileName) => transformAsyncSource(source, fileName, false), + force, "generators", )); + const changedFiles = generatedFiles.filter(file => file !== undefined); + if (!changedFiles.length) { + console.log("Sync API is up to date."); + return; + } console.log("Formatting..."); - formatFiles(generatedFiles); + formatFilesSync(changedFiles.map(file => file.fileName)); + for (const file of changedFiles) file.markCurrent(); console.log("Done."); } if (process.argv[1] === import.meta.filename) { - generateSync(); + generateSync(parseGeneratorArgs({}).force); } diff --git a/tools/scripts/gen/generateAPI.mts b/tools/scripts/gen/generateAPI.mts new file mode 100644 index 0000000000000..43a08bb8791ed --- /dev/null +++ b/tools/scripts/gen/generateAPI.mts @@ -0,0 +1,38 @@ +import path from "node:path"; +import { GeneratedFile } from "./generatedFile.mts"; +import { + formatFiles, + globInputs, + goInputs, + parseGeneratorArgs, + repoRoot, + runGo, +} from "./utils.mts"; + +export default async function generateAPI(force = false) { + const inputs = globInputs([ + "tsc/internal/api/*.go", + "tsc/internal/api/requestfilesystem/*.go", + "tsc/internal/core/*.go", + "tsc/internal/checker/types.go", + "tsc/internal/diagnostics/diagnostics.go", + "tsc/internal/tspath/path.go", + "tools/gen-proto/*.go", + ], ["**/*_test.go", "**/*_generated.go"]); + const generated = new GeneratedFile(path.join(repoRoot, "packages/typescript/src/api/proto.generated.ts"), [import.meta.filename, ...goInputs(), ...inputs]); + if (generated.isCurrent(force)) { + console.log("API protocol is up to date."); + return; + } + + generated.invalidate(); + console.log("Generating API protocol..."); + await runGo(["-C", "./tools", "run", "./gen-proto", "../tsc/internal/api/proto.go", generated.fileName]); + await formatFiles([generated.fileName]); + generated.markCurrent(); + console.log("Generated API protocol."); +} + +if (process.argv[1] === import.meta.filename) { + await generateAPI(parseGeneratorArgs({}).force); +} diff --git a/tools/scripts/gen/generateBundled.mts b/tools/scripts/gen/generateBundled.mts new file mode 100644 index 0000000000000..d07f5a53a1038 --- /dev/null +++ b/tools/scripts/gen/generateBundled.mts @@ -0,0 +1,40 @@ +import * as fs from "node:fs"; +import path from "node:path"; +import { GeneratedFile } from "./generatedFile.mts"; +import { + goEnvironment, + goInputs, + parseGeneratorArgs, + repoRoot, + runGo, +} from "./utils.mts"; + +const sourceDirectory = path.join(repoRoot, "tsc/internal/bundled"); + +export default async function generateBundled(force = false, directory = sourceDirectory) { + directory = path.resolve(directory); + const entries = fs.readdirSync(path.join(directory, "libs"), { withFileTypes: true }).sort((left, right) => left.name < right.name ? -1 : left.name > right.name ? 1 : 0); + const inputs = [ + import.meta.filename, + path.join(sourceDirectory, "generate.go"), + path.join(directory, "CopyrightNotice.txt"), + ...entries.filter(entry => !entry.isDirectory()).map(entry => path.join(directory, "libs", entry.name)), + ...goInputs(), + ]; + const key = [goEnvironment(), entries.map(entry => [entry.name, entry.isDirectory()])]; + const generatedFiles = ["libs_generated.go", "embed_generated.go"].map(file => new GeneratedFile(path.join(directory, file), inputs, undefined, key)); + if (generatedFiles.every(file => file.isCurrent(force))) { + console.log("Bundled libraries are up to date."); + return; + } + + for (const file of generatedFiles) file.invalidate(); + await runGo(["run", path.join(sourceDirectory, "generate.go")], directory); + for (const file of generatedFiles) file.markCurrent(); + console.log("Generated bundled libraries."); +} + +if (process.argv[1] === import.meta.filename) { + const { values, force } = parseGeneratorArgs({ directory: { type: "string" } }); + await generateBundled(force, values.directory); +} diff --git a/tools/scripts/gen/generateDiagnostics.mts b/tools/scripts/gen/generateDiagnostics.mts new file mode 100644 index 0000000000000..61f16d86c924e --- /dev/null +++ b/tools/scripts/gen/generateDiagnostics.mts @@ -0,0 +1,55 @@ +import * as fs from "node:fs"; +import path from "node:path"; +import { GeneratedFile } from "./generatedFile.mts"; +import { + formatFiles, + globInputs, + goEnvironment, + goInputs, + parseGeneratorArgs, + repoRoot, + runGo, +} from "./utils.mts"; + +const sourceDirectory = path.join(repoRoot, "tsc/internal/diagnostics"); + +export default async function generateDiagnostics(force = false, outDir = sourceDirectory) { + outDir = path.resolve(outDir); + const inputs = [ + import.meta.filename, + path.join(sourceDirectory, "generate.go"), + path.join(sourceDirectory, "diagnosticMessages.json"), + path.join(sourceDirectory, "extraDiagnosticMessages.json"), + ...goInputs(), + ...globInputs([ + "tsc/internal/{collections,json}/*.go", + "tsc/internal/locale/lcl/*/diagnosticMessages/diagnosticMessages.generated.json.lcl", + ], ["**/*_test.go"]), + ]; + const environment = goEnvironment(); + const generated = new GeneratedFile(path.join(outDir, "diagnostics_generated.go"), inputs, undefined, environment); + const localized = new GeneratedFile(path.join(outDir, "loc_generated.go"), inputs, undefined, environment); + const goFiles = [generated, localized]; + const localeFiles = () => + [...fs.readFileSync(localized.fileName, "utf8").matchAll(/^\/\/go:embed (loc\/[^\r\n]+\.json\.gz)\r?$/gm)] + .map(match => new GeneratedFile(path.join(outDir, match[1]), inputs, undefined, environment)); + if (goFiles.every(file => file.isCurrent(force))) { + const locales = localeFiles(); + if (locales.length === fs.globSync("loc/*.json.gz", { cwd: outDir }).length && locales.every(file => file.isCurrent())) { + console.log("Diagnostics are up to date."); + return; + } + } + + for (const file of goFiles) file.invalidate(); + fs.mkdirSync(outDir, { recursive: true }); + await runGo(["run", "generate.go", "-diagnostics", generated.fileName, "-loc", localized.fileName, "-locdir", path.join(outDir, "loc")], sourceDirectory); + await formatFiles(goFiles.map(file => file.fileName)); + for (const file of [...localeFiles(), ...goFiles]) file.markCurrent(); + console.log("Generated diagnostics."); +} + +if (process.argv[1] === import.meta.filename) { + const { values, force } = parseGeneratorArgs({ outDir: { type: "string" } }); + await generateDiagnostics(force, values.outDir); +} diff --git a/tools/scripts/gen/generateMoq.mts b/tools/scripts/gen/generateMoq.mts new file mode 100644 index 0000000000000..16f370fc18c08 --- /dev/null +++ b/tools/scripts/gen/generateMoq.mts @@ -0,0 +1,47 @@ +import path from "node:path"; +import { GeneratedFile } from "./generatedFile.mts"; +import { + formatFiles, + goEnvironment, + goInputs, + parseGeneratorArgs, + runGo, +} from "./utils.mts"; + +export default async function generateMoq(args: string[], cwd = process.cwd()) { + const { values, positionals, toolArgs: moqArgs, force } = parseGeneratorArgs( + { + out: { type: "string" }, + pkg: { type: "string" }, + fmt: { type: "string" }, + stub: { type: "boolean" }, + input: { type: "string", multiple: true }, + }, + args, + true, + ); + if (!values.out || !values.input?.length || positionals.length < 2) { + throw new Error("Usage: generateMoq.mts --input=source.go -out=mock.go [moq options] [--force] "); + } + const inputs = [ + import.meta.filename, + ...goInputs(), + ...values.input.map(file => path.resolve(cwd, file)), + ...(process.env.GOFILE ? [path.resolve(cwd, process.env.GOFILE)] : []), + ]; + const generated = new GeneratedFile(path.resolve(cwd, values.out), inputs, undefined, [moqArgs, goEnvironment()]); + if (generated.isCurrent(force)) { + console.log("Moq output is up to date."); + return; + } + + generated.invalidate(); + await runGo(["tool", "github.com/matryer/moq", ...moqArgs], cwd); + await formatFiles([generated.fileName]); + generated.markCurrent(); + console.log("Generated moq output."); +} + +if (process.argv[1] === import.meta.filename) { + await generateMoq(process.argv.slice(2)); +} diff --git a/tools/scripts/gen/generateStringer.mts b/tools/scripts/gen/generateStringer.mts new file mode 100644 index 0000000000000..cfe67017cf9e3 --- /dev/null +++ b/tools/scripts/gen/generateStringer.mts @@ -0,0 +1,49 @@ +import path from "node:path"; +import { GeneratedFile } from "./generatedFile.mts"; +import { + formatFiles, + goEnvironment, + goInputs, + parseGeneratorArgs, + runGo, +} from "./utils.mts"; + +export default async function generateStringer(args: string[], cwd = process.cwd()) { + const { values, positionals, toolArgs: stringerArgs, force } = parseGeneratorArgs( + { + type: { type: "string" }, + output: { type: "string" }, + trimprefix: { type: "string" }, + linecomment: { type: "boolean" }, + tags: { type: "string" }, + input: { type: "string" }, + }, + args, + true, + ); + const input = values.input ?? process.env.GOFILE; + if (!values.type || !values.output || !input || positionals.length) { + throw new Error("Usage: generateStringer.mts -type=Type -output=file.go [--input=source.go] [--force] (input defaults to GOFILE)"); + } + const output = path.resolve(cwd, values.output); + const inputs = [ + import.meta.filename, + ...goInputs(), + path.resolve(cwd, input), + ]; + const generated = new GeneratedFile(output, inputs, undefined, [stringerArgs, goEnvironment()]); + if (generated.isCurrent(force)) { + console.log("Stringer output is up to date."); + return; + } + + generated.invalidate(); + await runGo(["tool", "golang.org/x/tools/cmd/stringer", ...stringerArgs], cwd); + await formatFiles([output]); + generated.markCurrent(); + console.log("Generated stringer output."); +} + +if (process.argv[1] === import.meta.filename) { + await generateStringer(process.argv.slice(2)); +} diff --git a/tools/scripts/gen/generatedFile.mts b/tools/scripts/gen/generatedFile.mts new file mode 100644 index 0000000000000..b250b91deedc7 --- /dev/null +++ b/tools/scripts/gen/generatedFile.mts @@ -0,0 +1,78 @@ +import { createHash } from "node:crypto"; +import * as fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { repoRoot } from "./utils.mts"; + +const commonInputs = [ + import.meta.filename, + path.join(import.meta.dirname, "utils.mts"), + path.join(repoRoot, "package-lock.json"), + path.join(repoRoot, ".dprint.jsonc"), +]; + +function digest(value: string | Buffer): string { + return createHash("sha256").update(value).digest("hex"); +} + +function readIfExists(file: string): Buffer | undefined { + try { + return fs.readFileSync(file); + } + catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } +} + +export class GeneratedFile { + readonly fileName: string; + private readonly inputs: readonly string[]; + private readonly inputHash: string; + private readonly cacheFile: string; + private readonly key: unknown; + + constructor(fileName: string, inputs: readonly string[], cacheDirectory = path.join(os.tmpdir(), "typescript-codegen"), key?: unknown) { + this.fileName = path.resolve(fileName); + this.inputs = [...new Set([...commonInputs, ...inputs].map(file => path.resolve(file)))].sort(); + this.key = key; + this.inputHash = this.hashInputs(); + this.cacheFile = path.join(cacheDirectory, digest(this.fileName) + ".json"); + } + + private hashInputs(): string { + return digest(JSON.stringify([ + process.version, + process.platform, + process.arch, + this.key, + this.inputs.map(file => [file, digest(fs.readFileSync(file))]), + ])); + } + + private state(output: Buffer): string { + return JSON.stringify([this.inputHash, digest(output)]); + } + + isCurrent(force = false): boolean { + if (force) return false; + const output = readIfExists(this.fileName); + return output !== undefined && readIfExists(this.cacheFile)?.toString() === this.state(output); + } + + invalidate(): void { + fs.rmSync(this.cacheFile, { force: true }); + } + + write(content: string): void { + this.invalidate(); + fs.mkdirSync(path.dirname(this.fileName), { recursive: true }); + fs.writeFileSync(this.fileName, content); + } + + markCurrent(): void { + if (this.inputHash !== this.hashInputs()) return; + const state = this.state(fs.readFileSync(this.fileName)); + fs.mkdirSync(path.dirname(this.cacheFile), { recursive: true }); + fs.writeFileSync(this.cacheFile, state); + } +} diff --git a/tools/scripts/gen/generatedFile.test.mts b/tools/scripts/gen/generatedFile.test.mts new file mode 100644 index 0000000000000..a53c85a5eb282 --- /dev/null +++ b/tools/scripts/gen/generatedFile.test.mts @@ -0,0 +1,488 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import * as fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { x } from "tinyexec"; +import { GeneratedFile } from "./generatedFile.mts"; + +test("generator options preserve tool arguments and force overrides", async () => { + const { parseGeneratorArgs, resolveForce } = await import("./utils.mts"); + const options = { type: { type: "string" }, input: { type: "string", multiple: true } } as const; + const args = ["-type", "Kind", "--input=kind.go", "--input", "extra.go", "--force", ".", "Kind:Mock"]; + const parsed = parseGeneratorArgs(options, args, true); + assert.equal(parsed.values.type, "Kind"); + assert.deepEqual(parsed.values.input, ["kind.go", "extra.go"]); + assert.deepEqual(parsed.positionals, [".", "Kind:Mock"]); + assert.deepEqual(parsed.toolArgs, ["-type", "Kind", ".", "Kind:Mock"]); + assert.equal(parsed.force, true); + assert.equal(resolveForce(undefined, "ON"), true); + assert.equal(resolveForce(undefined, "0"), false); + assert.equal(resolveForce(false, "1"), false); + assert.equal(parseGeneratorArgs({}, ["--no-force"]).force, false); +}); + +test("generated files are current only while their inputs and formatted output match", context => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-codegen-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const input = path.join(directory, "input.ts"); + const output = path.join(directory, "output.ts"); + const cache = path.join(directory, "cache"); + fs.writeFileSync(input, "input"); + const generated = () => new GeneratedFile(output, [input], cache); + let file = generated(); + assert.equal(file.isCurrent(), false); + file.write("unformatted"); + fs.writeFileSync(output, "formatted"); + file.markCurrent(); + assert.equal(generated().isCurrent(), true); + assert.equal(generated().isCurrent(true), false); + file.invalidate(); + assert.equal(generated().isCurrent(), false); + file.write("formatted"); + file.markCurrent(); + assert.equal(generated().isCurrent(), true); + fs.utimesSync(input, new Date(), new Date()); + assert.equal(generated().isCurrent(), true); + fs.writeFileSync(input, "changed"); + assert.equal(generated().isCurrent(), false); + fs.writeFileSync(input, "input"); + assert.equal(generated().isCurrent(), true); + fs.writeFileSync(output, "edited"); + assert.equal(generated().isCurrent(), false); + file = generated(); + file.write("formatted"); + assert.equal(generated().isCurrent(), false); + file.markCurrent(); + assert.equal(generated().isCurrent(), true); + const dependency = path.join(directory, "generator.ts"); + fs.writeFileSync(dependency, "generator"); + assert.equal(new GeneratedFile(output, [input, dependency], cache).isCurrent(), false); + fs.rmSync(dependency); + assert.throws(() => new GeneratedFile(output, [input, dependency], cache), { code: "ENOENT" }); + fs.rmSync(output); + assert.equal(generated().isCurrent(), false); +}); + +test("changing inputs during generation does not mark stale output current", context => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-codegen-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const input = path.join(directory, "input.ts"); + const output = path.join(directory, "output.ts"); + const cache = path.join(directory, "cache"); + fs.writeFileSync(input, "before"); + const file = new GeneratedFile(output, [input], cache); + file.write("generated"); + fs.writeFileSync(input, "after"); + file.markCurrent(); + assert.equal(new GeneratedFile(output, [input], cache).isCurrent(), false); +}); + +test("default metadata has a stable path in the OS temporary directory", context => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-codegen-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const output = path.join(directory, "output.ts"); + const cacheFile = path.join(os.tmpdir(), "typescript-codegen", createHash("sha256").update(output).digest("hex") + ".json"); + context.after(() => fs.rmSync(cacheFile, { force: true })); + const file = new GeneratedFile(output, []); + file.write("generated"); + file.markCurrent(); + assert.equal(fs.existsSync(cacheFile), true); + assert.equal(new GeneratedFile(output, []).isCurrent(), true); + fs.rmSync(cacheFile); + assert.equal(new GeneratedFile(output, []).isCurrent(), false); + file.markCurrent(); + assert.equal(new GeneratedFile(output, []).isCurrent(), true); +}); + +test("metadata directories can disappear during generation", context => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-codegen-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const output = path.join(directory, "output.ts"); + const cache = path.join(directory, "cache"); + const file = new GeneratedFile(output, [], cache); + file.write("generated"); + file.markCurrent(); + fs.rmSync(cache, { recursive: true }); + assert.equal(file.isCurrent(), false); + file.write("regenerated"); + file.markCurrent(); + assert.equal(new GeneratedFile(output, [], cache).isCurrent(), true); +}); + +test("generated files track each output independently", context => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-codegen-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const files = ["output.go", "locale.gz"].map(file => new GeneratedFile(path.join(directory, file), [], path.join(directory, "cache"))); + for (const file of files) { + file.write("generated"); + file.markCurrent(); + } + assert.equal(files.every(file => file.isCurrent()), true); + fs.writeFileSync(files[1].fileName, "modified"); + assert.equal(files[0].isCurrent(), true); + assert.equal(files[1].isCurrent(), false); + fs.writeFileSync(files[1].fileName, "generated"); + assert.equal(files.every(file => file.isCurrent()), true); + fs.rmSync(files[1].fileName); + assert.equal(files[0].isCurrent(), true); + assert.equal(files[1].isCurrent(), false); + files[1].write("generated"); + files[1].markCurrent(); + files[1].invalidate(); + assert.equal(files[0].isCurrent(), true); + assert.equal(files[1].isCurrent(), false); +}); + +test("bundled generation skips unchanged library outputs", async () => { + const root = path.resolve(import.meta.dirname, "../../.."); + const generate = () => x("go", ["-C", "./tsc", "generate", "./internal/bundled"], { throwOnError: true, nodeOptions: { cwd: root } }); + await generate(); + const files = ["libs_generated.go", "embed_generated.go"].map(file => path.join(root, "tsc/internal/bundled", file)); + const timestamps = files.map(file => fs.statSync(file).mtimeMs); + assert.match((await generate()).stdout, /Bundled libraries are up to date\./); + assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); +}); + +test("bundled generation tracks libraries, validates inputs, and supports force", async context => { + const root = path.resolve(import.meta.dirname, "../../.."); + const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_bundled-probe-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true, maxRetries: 10 })); + const notice = path.join(directory, "CopyrightNotice.txt"); + const libs = path.join(directory, "libs"); + fs.mkdirSync(libs); + fs.writeFileSync(notice, "Fixture notice\n\n"); + const library = path.join(libs, "lib.first.d.ts"); + const content = "Fixture notice\n\n\ndeclare const first: string;\n"; + fs.writeFileSync(library, content); + const command = [path.join(root, "tools/scripts/gen/generateBundled.mts"), "--directory", directory]; + const generate = (...args: string[]) => x(process.execPath, [...command, ...args], { throwOnError: true, nodeOptions: { cwd: root } }); + await generate(); + const files = ["libs_generated.go", "embed_generated.go"].map(file => path.join(directory, file)); + const originals = files.map(file => fs.readFileSync(file)); + assert.match((await generate()).stdout, /Bundled libraries are up to date\./); + const added = path.join(libs, "lib.second.d.ts"); + fs.writeFileSync(added, content); + assert.match((await generate()).stdout, /Generated bundled libraries\./); + for (const file of files) assert.match(fs.readFileSync(file, "utf8"), /lib\.second\.d\.ts/); + fs.rmSync(added); + assert.match((await generate()).stdout, /Generated bundled libraries\./); + for (const [index, file] of files.entries()) { + assert.deepEqual(fs.readFileSync(file), originals[index]); + fs.writeFileSync(file, "modified"); + assert.match((await generate()).stdout, /Generated bundled libraries\./); + assert.deepEqual(fs.readFileSync(file), originals[index]); + fs.rmSync(file, { recursive: true, maxRetries: 10 }); + assert.match((await generate()).stdout, /Generated bundled libraries\./); + assert.deepEqual(fs.readFileSync(file), originals[index]); + } + fs.writeFileSync(library, content.replace("string", "number")); + assert.match((await generate()).stdout, /Generated bundled libraries\./); + fs.writeFileSync(library, content.replaceAll("\n", "\r\n")); + await assert.rejects(async () => await generate(), error => { + assert.match((error as { output: { stderr: string; }; }).output.stderr, /must use LF line endings/); + return true; + }); + fs.writeFileSync(library, content); + assert.match((await generate()).stdout, /Generated bundled libraries\./); + fs.mkdirSync(path.join(libs, "unexpected")); + await assert.rejects(async () => await generate(), error => { + assert.match((error as { output: { stderr: string; }; }).output.stderr, /unexpected entry/); + return true; + }); + fs.rmdirSync(path.join(libs, "unexpected")); + assert.match((await generate()).stdout, /Generated bundled libraries\./); + fs.writeFileSync(notice, "Changed fixture notice\n\n"); + await assert.rejects(async () => await generate(), error => { + assert.match((error as { output: { stderr: string; }; }).output.stderr, /must start with/); + return true; + }); + fs.writeFileSync(notice, "Fixture notice\n\n"); + assert.match((await generate()).stdout, /Generated bundled libraries\./); + assert.match((await generate("--force")).stdout, /Generated bundled libraries\./); + assert.match((await generate()).stdout, /Bundled libraries are up to date\./); + const forced = await x(process.execPath, command, { + throwOnError: true, + nodeOptions: { cwd: root, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, + }); + assert.match(forced.stdout, /Generated bundled libraries\./); + assert.match((await generate()).stdout, /Bundled libraries are up to date\./); +}); + +test("stringer generation tracks GOFILE, options, output, and force", async context => { + const root = path.resolve(import.meta.dirname, "../../.."); + const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_stringer-probe-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const source = path.join(directory, "kind.go"); + const extra = path.join(directory, "extra.go"); + const output = path.join(directory, "kind_stringer_generated.go"); + fs.writeFileSync(source, "package probe\n\ntype Kind int\n\nconst KindFirst Kind = 0\n"); + const generate = (...args: string[]) => + x(process.execPath, [path.join(root, "tools/scripts/gen/generateStringer.mts"), "-type=Kind", "-output=kind_stringer_generated.go", ...args], { + throwOnError: true, + nodeOptions: { cwd: directory, env: { ...process.env, GOFILE: "kind.go" } }, + }); + await generate(); + const timestamp = fs.statSync(output).mtimeMs; + assert.match((await generate()).stdout, /Stringer output is up to date\./); + assert.equal(fs.statSync(output).mtimeMs, timestamp); + fs.writeFileSync(extra, "package probe\n\nconst Unrelated = 1\n"); + assert.match((await generate()).stdout, /Stringer output is up to date\./); + fs.appendFileSync(source, "\nconst KindSecond Kind = 1\n"); + assert.match((await generate()).stdout, /Generated stringer output\./); + assert.match(fs.readFileSync(output, "utf8"), /KindSecond/); + fs.rmSync(extra); + assert.match((await generate()).stdout, /Stringer output is up to date\./); + fs.writeFileSync(source, "package probe\n\ntype Kind int\n\nconst KindFirst Kind = 0\n"); + assert.match((await generate()).stdout, /Generated stringer output\./); + assert.doesNotMatch(fs.readFileSync(output, "utf8"), /KindSecond/); + assert.match((await generate("-trimprefix=Kind")).stdout, /Generated stringer output\./); + assert.match(fs.readFileSync(output, "utf8"), /_Kind_name = "First"/); + assert.match((await generate("-trimprefix=Kind")).stdout, /Stringer output is up to date\./); + fs.appendFileSync(output, "\n"); + assert.match((await generate("-trimprefix=Kind")).stdout, /Generated stringer output\./); + fs.rmSync(output); + assert.match((await generate("-trimprefix=Kind")).stdout, /Generated stringer output\./); + assert.match((await generate("-trimprefix=Kind", "--force")).stdout, /Generated stringer output\./); + assert.match((await generate("-trimprefix=Kind")).stdout, /Stringer output is up to date\./); + const command = [path.join(root, "tools/scripts/gen/generateStringer.mts"), "-type=Kind", "-output=kind_stringer_generated.go", "-trimprefix=Kind"]; + const explicit = await x(process.execPath, [...command, "--input", "kind.go"], { + throwOnError: true, + nodeOptions: { cwd: directory, env: { ...process.env, GOFILE: "" } }, + }); + assert.match(explicit.stdout, /Stringer output is up to date\./); + const forced = await x(process.execPath, command, { + throwOnError: true, + nodeOptions: { cwd: directory, env: { ...process.env, GOFILE: "kind.go", TSGO_HEREBY_FORCE: "1" } }, + }); + assert.match(forced.stdout, /Generated stringer output\./); + assert.match((await generate("-trimprefix=Kind")).stdout, /Stringer output is up to date\./); + fs.appendFileSync(source, "\nconst KindInvalid Kind = missing\n"); + await assert.rejects(async () => await generate("-trimprefix=Kind")); + fs.writeFileSync(source, "package probe\n\ntype Kind int\n\nconst KindFirst Kind = 0\n"); + assert.match((await generate("-trimprefix=Kind")).stdout, /Generated stringer output\./); + assert.match((await generate("-trimprefix=Kind")).stdout, /Stringer output is up to date\./); +}); + +test("moq generation tracks interface inputs, output, and force", async context => { + const root = path.resolve(import.meta.dirname, "../../.."); + const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_moq-probe-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const source = path.join(directory, "client.go"); + const base = path.join(directory, "base.go"); + fs.mkdirSync(path.join(directory, "mock")); + const output = path.join(directory, "mock/mock_generated.go"); + fs.writeFileSync(base, "package probe\n\ntype Base interface { Read() string }\n"); + fs.writeFileSync(source, "package probe\n\ntype Client interface { Base }\n"); + const command = [path.join(root, "tools/scripts/gen/generateMoq.mts"), "--input=client.go", "--input=base.go", "-stub", "-fmt=goimports", "-pkg=mock", "-out=mock/mock_generated.go"]; + const generate = (...args: string[]) => + x(process.execPath, [...command, ...args, ".", "Client"], { + throwOnError: true, + nodeOptions: { cwd: directory }, + }); + await generate(); + const timestamp = fs.statSync(output).mtimeMs; + assert.match((await generate()).stdout, /Moq output is up to date\./); + assert.equal(fs.statSync(output).mtimeMs, timestamp); + fs.writeFileSync(base, "package probe\n\ntype Base interface { Read() string; Close() error }\n"); + assert.match((await generate()).stdout, /Generated moq output\./); + assert.match(fs.readFileSync(output, "utf8"), /CloseFunc/); + fs.rmSync(output); + assert.match((await generate()).stdout, /Generated moq output\./); + assert.match((await generate("--force")).stdout, /Generated moq output\./); + assert.match((await generate()).stdout, /Moq output is up to date\./); + const forced = await x(process.execPath, [...command, ".", "Client"], { + throwOnError: true, + nodeOptions: { cwd: directory, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, + }); + assert.match(forced.stdout, /Generated moq output\./); + assert.match((await generate()).stdout, /Moq output is up to date\./); +}); + +test("diagnostic generation tracks Go and locale outputs and supports force", async context => { + const root = path.resolve(import.meta.dirname, "../../.."); + const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_diagnostics-probe-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const generate = (...args: string[]) => + x(process.execPath, [path.join(root, "tools/scripts/gen/generateDiagnostics.mts"), "--outDir", directory, ...args], { + throwOnError: true, + nodeOptions: { cwd: root }, + }); + await generate(); + const output = path.join(directory, "diagnostics_generated.go"); + const localized = path.join(directory, "loc_generated.go"); + const locales = fs.globSync("loc/*.json.gz", { cwd: directory }).map(file => path.join(directory, file)); + assert.ok(locales.length > 0); + const files = [output, localized, ...locales]; + const timestamps = files.map(file => fs.statSync(file).mtimeMs); + assert.match((await generate()).stdout, /Diagnostics are up to date\./); + assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); + const archive = fs.readFileSync(locales[0]); + fs.rmSync(locales[0]); + assert.match((await generate()).stdout, /Generated diagnostics\./); + assert.deepEqual(fs.readFileSync(locales[0]), archive); + fs.writeFileSync(locales[0], "modified"); + assert.match((await generate()).stdout, /Generated diagnostics\./); + assert.deepEqual(fs.readFileSync(locales[0]), archive); + const unexpected = path.join(directory, "loc/unexpected.json.gz"); + fs.writeFileSync(unexpected, "unexpected"); + assert.match((await generate()).stdout, /Generated diagnostics\./); + assert.equal(fs.existsSync(unexpected), false); + fs.rmSync(path.join(directory, "loc"), { recursive: true }); + assert.match((await generate()).stdout, /Generated diagnostics\./); + assert.deepEqual(fs.readFileSync(locales[0]), archive); + fs.rmSync(localized); + assert.match((await generate()).stdout, /Generated diagnostics\./); + assert.match((await generate("--force")).stdout, /Generated diagnostics\./); + assert.match((await generate()).stdout, /Diagnostics are up to date\./); + const forced = await x(process.execPath, [path.join(root, "tools/scripts/gen/generateDiagnostics.mts"), "--outDir", directory], { + throwOnError: true, + nodeOptions: { cwd: root, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, + }); + assert.match(forced.stdout, /Generated diagnostics\./); + assert.match((await generate()).stdout, /Diagnostics are up to date\./); +}); + +test("Unicode generation skips unchanged tables and formatting", async () => { + const root = path.resolve(import.meta.dirname, "../../.."); + const generate = () => x("go", ["-C", "./tsc", "generate", "./internal/stringutil"], { throwOnError: true, nodeOptions: { cwd: root } }); + await generate(); + const files = ["js_case_generated.go", "identifier_parts_generated.go"].map(file => path.join(root, "tsc/internal/stringutil", file)); + const timestamps = files.map(file => fs.statSync(file).mtimeMs); + const current = await generate(); + assert.match(current.stdout, /Unicode tables are up to date\./); + assert.doesNotMatch(current.stdout, /Formatted/); + assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); +}); + +test("Unicode generation repairs outputs and supports force", async context => { + const root = path.resolve(import.meta.dirname, "../../.."); + const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_unicode-probe-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const command = [path.join(root, "tsc/internal/stringutil/_scripts/generate-unicode-data.mts"), "--outDir", directory]; + const generate = (...args: string[]) => x(process.execPath, [...command, ...args], { throwOnError: true, nodeOptions: { cwd: root } }); + await generate(); + const files = ["js_case_generated.go", "identifier_parts_generated.go"].map(file => path.join(directory, file)); + const originals = files.map(file => fs.readFileSync(file)); + const timestamps = files.map(file => fs.statSync(file).mtimeMs); + assert.match((await generate()).stdout, /Unicode tables are up to date\./); + assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); + for (const [index, file] of files.entries()) { + assert.deepEqual(originals[index], fs.readFileSync(path.join(root, "tsc/internal/stringutil", path.basename(file)))); + fs.writeFileSync(file, "modified"); + assert.match((await generate()).stdout, /Generated Unicode tables\./); + assert.deepEqual(fs.readFileSync(file), originals[index]); + fs.rmSync(file); + assert.match((await generate()).stdout, /Generated Unicode tables\./); + assert.deepEqual(fs.readFileSync(file), originals[index]); + } + const cacheFile = path.join(os.tmpdir(), "typescript-codegen", createHash("sha256").update(files[0]).digest("hex") + ".json"); + context.after(() => fs.rmSync(cacheFile, { force: true })); + fs.rmSync(cacheFile); + assert.match((await generate()).stdout, /Generated Unicode tables\./); + assert.match((await generate("--force")).stdout, /Generated Unicode tables\./); + assert.match((await generate()).stdout, /Unicode tables are up to date\./); + const forced = await x(process.execPath, command, { + throwOnError: true, + nodeOptions: { cwd: root, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, + }); + assert.match(forced.stdout, /Generated Unicode tables\./); + const overridden = await x(process.execPath, [...command, "--no-force"], { + throwOnError: true, + nodeOptions: { cwd: root, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, + }); + assert.match(overridden.stdout, /Unicode tables are up to date\./); +}); + +test("enum generation skips unchanged outputs and Go verification", async () => { + const root = path.resolve(import.meta.dirname, "../../.."); + const generate = (force = false) => x("npx", ["hereby", "generate:enums", ...(force ? ["--force"] : [])], { throwOnError: true, nodeOptions: { cwd: root } }); + await generate(); + const files = [ + ...fs.globSync("packages/typescript/src/enums/*.ts", { cwd: root }), + "tsc/internal/api/enum_values_generated.go", + ].map(file => path.join(root, file)); + const timestamps = files.map(file => fs.statSync(file).mtimeMs); + const { stdout } = await generate(); + assert.match(stdout, /Enums are up to date\./); + assert.doesNotMatch(stdout, /Getting values from go/); + assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); + const forced = await generate(true); + assert.match(forced.stdout, /All generated values match Go\./); + assert.match((await generate()).stdout, /Enums are up to date\./); + const verifier = path.join(root, "tsc/internal/api/enum_values_generated.go"); + const cacheFile = path.join(os.tmpdir(), "typescript-codegen", createHash("sha256").update(verifier).digest("hex") + ".json"); + fs.rmSync(cacheFile); + const regenerated = await generate(); + assert.match(regenerated.stdout, /All generated values match Go\./); + assert.match((await generate()).stdout, /Enums are up to date\./); +}); + +test("AST generation forwards force to all schema generators", async () => { + const root = path.resolve(import.meta.dirname, "../../.."); + const generate = (force = false) => x("npx", ["hereby", "generate:ast", ...(force ? ["--force"] : [])], { throwOnError: true, nodeOptions: { cwd: root } }); + await generate(); + const forced = await generate(true); + assert.match(forced.stdout, /Wrote .*encoder_generated\.go/); + assert.match(forced.stdout, /Wrote .*ast_generated\.go/); + assert.match(forced.stdout, /Generated .*ast\.generated\.ts/); + assert.doesNotMatch(forced.stdout, /\$ node .*tools\/scripts\/tsc\/generate\.ts/); + assert.doesNotMatch((await generate()).stdout, /(?:Wrote|Generated) /); +}); + +test("package generation forwards force to AST, encoder, and sync generators", async () => { + const root = path.resolve(import.meta.dirname, "../../.."); + const generate = (force = false) => x("npm", ["run", "-w", "@typescript/typescript", "generate", ...(force ? ["--", "--force"] : [])], { throwOnError: true, nodeOptions: { cwd: root } }); + await generate(); + const forced = await generate(true); + assert.match(forced.stdout, /Generated .*ast\.generated\.ts/); + assert.match(forced.stdout, /Wrote .*encoder_generated\.go/); + assert.match(forced.stdout, /Formatting\.\.\./); + const current = await generate(); + assert.doesNotMatch(current.stdout, /(?:Wrote|Generated) |Formatting\.\.\./); + assert.match(current.stdout, /Sync API is up to date\./); +}); + +test("API protocol generation caches formatted output and supports force", async context => { + const root = path.resolve(import.meta.dirname, "../../.."); + const output = path.join(root, "packages/typescript/src/api/proto.generated.ts"); + const generate = (force = false) => x("npx", ["hereby", "generate:api", ...(force ? ["--force"] : [])], { throwOnError: true, nodeOptions: { cwd: root } }); + await generate(); + const timestamp = fs.statSync(output).mtimeMs; + const current = await generate(); + assert.match(current.stdout, /API protocol is up to date\./); + assert.doesNotMatch(current.stdout, /\$ node .*generateAPI\.mts/); + assert.equal(fs.statSync(output).mtimeMs, timestamp); + const unrelated = path.join(root, "tsc/internal/parser/codegen_cache_probe.go"); + assert.equal(fs.existsSync(unrelated), false); + context.after(() => fs.rmSync(unrelated, { force: true })); + fs.writeFileSync(unrelated, "package parser\n"); + assert.match((await generate()).stdout, /API protocol is up to date\./); + fs.rmSync(unrelated); + const schema = path.join(root, "tsc/internal/api/requestfilesystem/codegen_cache_probe.go"); + assert.equal(fs.existsSync(schema), false); + const original = fs.readFileSync(output, "utf8"); + context.after(async () => { + if (fs.existsSync(schema)) { + fs.rmSync(schema); + await generate(true); + } + }); + fs.writeFileSync(schema, 'package requestfilesystem\n\nconst KindCodegenCacheProbe Kind = "codegen-cache-probe"\n'); + assert.match((await generate()).stdout, /Generated API protocol\./); + assert.match(fs.readFileSync(output, "utf8"), /"codegen-cache-probe"/); + fs.rmSync(schema); + assert.match((await generate()).stdout, /Generated API protocol\./); + assert.equal(fs.readFileSync(output, "utf8"), original); + assert.match((await generate(true)).stdout, /Generated API protocol\./); + assert.match((await generate()).stdout, /API protocol is up to date\./); + const direct = await x("node", ["./tools/scripts/gen/generateAPI.mts"], { throwOnError: true, nodeOptions: { cwd: root } }); + assert.match(direct.stdout, /API protocol is up to date\./); + const nested = await x("go", ["-C", "./tsc", "generate", "./internal/api"], { + throwOnError: true, + nodeOptions: { cwd: root, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, + }); + assert.match(nested.stdout, /Generated API protocol\./); + assert.match((await generate()).stdout, /API protocol is up to date\./); +}); diff --git a/tools/scripts/gen/tsconfig.json b/tools/scripts/gen/tsconfig.json new file mode 100644 index 0000000000000..d4d71e52e8f38 --- /dev/null +++ b/tools/scripts/gen/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "module": "nodenext", + "lib": ["esnext"], + "noEmit": true, + "verbatimModuleSyntax": true, + "erasableSyntaxOnly": true, + "allowImportingTsExtensions": true, + "types": ["node"] + } +} diff --git a/tools/scripts/gen/utils.mts b/tools/scripts/gen/utils.mts new file mode 100644 index 0000000000000..86f455a65f90d --- /dev/null +++ b/tools/scripts/gen/utils.mts @@ -0,0 +1,56 @@ +import * as fs from "node:fs"; +import path from "node:path"; +import { + parseArgs, + type ParseArgsOptionsConfig, +} from "node:util"; +import { + x, + xSync, +} from "tinyexec"; + +export const repoRoot = path.resolve(import.meta.dirname, "../../.."); + +export function globInputs(patterns: string[], exclude: string[] = []): string[] { + return fs.globSync(patterns, { cwd: repoRoot, exclude }).map(file => path.join(repoRoot, file)); +} + +export function goInputs(): string[] { + return [import.meta.filename, ...globInputs(["go.work", "go.work.sum", "{tsc,tools}/go.{mod,sum}"])]; +} + +export function goEnvironment(): Record { + return Object.fromEntries(["GOOS", "GOARCH", "GOFLAGS", "GOTOOLCHAIN", "GOEXPERIMENT", "CGO_ENABLED", "GOWORK"].map(name => [name, process.env[name]])); +} + +export function resolveForce(force?: boolean, environment = process.env.TSGO_HEREBY_FORCE): boolean { + return force ?? /^(1|true|yes|on)$/i.test(environment ?? ""); +} + +export function parseGeneratorArgs(options: Options, args = process.argv.slice(2), goStyle = false) { + const { values, positionals, tokens } = parseArgs({ + args: goStyle ? args.map(arg => arg.replace(/^-(?=[a-z])/, "--")) : args, + options: { ...options, force: { type: "boolean" as const } }, + allowPositionals: goStyle, + allowNegative: true, + tokens: true, + }); + const toolArgs = tokens.flatMap(token => { + if (token.kind === "positional") return [token.value]; + if (token.kind !== "option" || ["force", "no-force", "input"].includes(token.name)) return []; + return args.slice(token.index, token.index + (token.value !== undefined && !token.inlineValue ? 2 : 1)); + }); + return { values, positionals, toolArgs, force: resolveForce((values as { force?: boolean; }).force) }; +} + +export async function runGo(args: string[], cwd = repoRoot): Promise { + await x("go", args, { throwOnError: true, nodeOptions: { cwd, stdio: "inherit" } }); +} + +export async function formatFiles(files: string[]): Promise { + await x("dprint", ["fmt", ...files], { throwOnError: true, nodeOptions: { cwd: repoRoot, stdio: "inherit" } }); +} + +export function formatFilesSync(files: string[]): void { + xSync("dprint", ["fmt", ...files], { throwOnError: true, nodeOptions: { cwd: repoRoot, stdio: "inherit" } }); +} diff --git a/tools/scripts/generatedFile.mts b/tools/scripts/generatedFile.mts new file mode 100644 index 0000000000000..d05521edc7c7c --- /dev/null +++ b/tools/scripts/generatedFile.mts @@ -0,0 +1,77 @@ +import { createHash } from "node:crypto"; +import * as fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const repoRoot = path.resolve(import.meta.dirname, "../.."); +const commonInputs = [ + import.meta.filename, + path.join(repoRoot, "package-lock.json"), + path.join(repoRoot, ".dprint.jsonc"), +]; + +function digest(value: string | Buffer): string { + return createHash("sha256").update(value).digest("hex"); +} + +function readIfExists(file: string): Buffer | undefined { + try { + return fs.readFileSync(file); + } + catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } +} + +export class GeneratedFile { + readonly fileName: string; + private readonly inputs: readonly string[]; + private readonly inputHash: string; + private readonly cacheFile: string; + private readonly key: unknown; + + constructor(fileName: string, inputs: readonly string[], cacheDirectory = path.join(os.tmpdir(), "typescript-codegen"), key?: unknown) { + this.fileName = path.resolve(fileName); + this.inputs = [...new Set([...commonInputs, ...inputs].map(file => path.resolve(file)))].sort(); + this.key = key; + this.inputHash = this.hashInputs(); + this.cacheFile = path.join(cacheDirectory, digest(this.fileName) + ".json"); + } + + private hashInputs(): string { + return digest(JSON.stringify([ + process.version, + process.platform, + process.arch, + this.key, + this.inputs.map(file => [file, digest(fs.readFileSync(file))]), + ])); + } + + private state(output: Buffer): string { + return JSON.stringify([this.inputHash, digest(output)]); + } + + isCurrent(force = false): boolean { + if (force) return false; + const output = readIfExists(this.fileName); + return output !== undefined && readIfExists(this.cacheFile)?.toString() === this.state(output); + } + + invalidate(): void { + fs.rmSync(this.cacheFile, { force: true }); + } + + write(content: string): void { + this.invalidate(); + fs.mkdirSync(path.dirname(this.fileName), { recursive: true }); + fs.writeFileSync(this.fileName, content); + } + + markCurrent(): void { + if (this.inputHash !== this.hashInputs()) return; + const state = this.state(fs.readFileSync(this.fileName)); + fs.mkdirSync(path.dirname(this.cacheFile), { recursive: true }); + fs.writeFileSync(this.cacheFile, state); + } +} diff --git a/tools/scripts/generatedFile.test.mts b/tools/scripts/generatedFile.test.mts new file mode 100644 index 0000000000000..2f510a6e2aa32 --- /dev/null +++ b/tools/scripts/generatedFile.test.mts @@ -0,0 +1,488 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import * as fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { x } from "tinyexec"; +import { GeneratedFile } from "./generatedFile.mts"; + +test("generator options preserve tool arguments and force overrides", async () => { + const { parseGeneratorArgs, resolveForce } = await import("./gen/utils.mts"); + const options = { type: { type: "string" }, input: { type: "string", multiple: true } } as const; + const args = ["-type", "Kind", "--input=kind.go", "--input", "extra.go", "--force", ".", "Kind:Mock"]; + const parsed = parseGeneratorArgs(options, args, true); + assert.equal(parsed.values.type, "Kind"); + assert.deepEqual(parsed.values.input, ["kind.go", "extra.go"]); + assert.deepEqual(parsed.positionals, [".", "Kind:Mock"]); + assert.deepEqual(parsed.toolArgs, ["-type", "Kind", ".", "Kind:Mock"]); + assert.equal(parsed.force, true); + assert.equal(resolveForce(undefined, "ON"), true); + assert.equal(resolveForce(undefined, "0"), false); + assert.equal(resolveForce(false, "1"), false); + assert.equal(parseGeneratorArgs({}, ["--no-force"]).force, false); +}); + +test("generated files are current only while their inputs and formatted output match", context => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-codegen-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const input = path.join(directory, "input.ts"); + const output = path.join(directory, "output.ts"); + const cache = path.join(directory, "cache"); + fs.writeFileSync(input, "input"); + const generated = () => new GeneratedFile(output, [input], cache); + let file = generated(); + assert.equal(file.isCurrent(), false); + file.write("unformatted"); + fs.writeFileSync(output, "formatted"); + file.markCurrent(); + assert.equal(generated().isCurrent(), true); + assert.equal(generated().isCurrent(true), false); + file.invalidate(); + assert.equal(generated().isCurrent(), false); + file.write("formatted"); + file.markCurrent(); + assert.equal(generated().isCurrent(), true); + fs.utimesSync(input, new Date(), new Date()); + assert.equal(generated().isCurrent(), true); + fs.writeFileSync(input, "changed"); + assert.equal(generated().isCurrent(), false); + fs.writeFileSync(input, "input"); + assert.equal(generated().isCurrent(), true); + fs.writeFileSync(output, "edited"); + assert.equal(generated().isCurrent(), false); + file = generated(); + file.write("formatted"); + assert.equal(generated().isCurrent(), false); + file.markCurrent(); + assert.equal(generated().isCurrent(), true); + const dependency = path.join(directory, "generator.ts"); + fs.writeFileSync(dependency, "generator"); + assert.equal(new GeneratedFile(output, [input, dependency], cache).isCurrent(), false); + fs.rmSync(dependency); + assert.throws(() => new GeneratedFile(output, [input, dependency], cache), { code: "ENOENT" }); + fs.rmSync(output); + assert.equal(generated().isCurrent(), false); +}); + +test("changing inputs during generation does not mark stale output current", context => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-codegen-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const input = path.join(directory, "input.ts"); + const output = path.join(directory, "output.ts"); + const cache = path.join(directory, "cache"); + fs.writeFileSync(input, "before"); + const file = new GeneratedFile(output, [input], cache); + file.write("generated"); + fs.writeFileSync(input, "after"); + file.markCurrent(); + assert.equal(new GeneratedFile(output, [input], cache).isCurrent(), false); +}); + +test("default metadata has a stable path in the OS temporary directory", context => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-codegen-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const output = path.join(directory, "output.ts"); + const cacheFile = path.join(os.tmpdir(), "typescript-codegen", createHash("sha256").update(output).digest("hex") + ".json"); + context.after(() => fs.rmSync(cacheFile, { force: true })); + const file = new GeneratedFile(output, []); + file.write("generated"); + file.markCurrent(); + assert.equal(fs.existsSync(cacheFile), true); + assert.equal(new GeneratedFile(output, []).isCurrent(), true); + fs.rmSync(cacheFile); + assert.equal(new GeneratedFile(output, []).isCurrent(), false); + file.markCurrent(); + assert.equal(new GeneratedFile(output, []).isCurrent(), true); +}); + +test("metadata directories can disappear during generation", context => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-codegen-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const output = path.join(directory, "output.ts"); + const cache = path.join(directory, "cache"); + const file = new GeneratedFile(output, [], cache); + file.write("generated"); + file.markCurrent(); + fs.rmSync(cache, { recursive: true }); + assert.equal(file.isCurrent(), false); + file.write("regenerated"); + file.markCurrent(); + assert.equal(new GeneratedFile(output, [], cache).isCurrent(), true); +}); + +test("generated files track each output independently", context => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-codegen-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const files = ["output.go", "locale.gz"].map(file => new GeneratedFile(path.join(directory, file), [], path.join(directory, "cache"))); + for (const file of files) { + file.write("generated"); + file.markCurrent(); + } + assert.equal(files.every(file => file.isCurrent()), true); + fs.writeFileSync(files[1].fileName, "modified"); + assert.equal(files[0].isCurrent(), true); + assert.equal(files[1].isCurrent(), false); + fs.writeFileSync(files[1].fileName, "generated"); + assert.equal(files.every(file => file.isCurrent()), true); + fs.rmSync(files[1].fileName); + assert.equal(files[0].isCurrent(), true); + assert.equal(files[1].isCurrent(), false); + files[1].write("generated"); + files[1].markCurrent(); + files[1].invalidate(); + assert.equal(files[0].isCurrent(), true); + assert.equal(files[1].isCurrent(), false); +}); + +test("bundled generation skips unchanged library outputs", async () => { + const root = path.resolve(import.meta.dirname, "../.."); + const generate = () => x("go", ["-C", "./tsc", "generate", "./internal/bundled"], { throwOnError: true, nodeOptions: { cwd: root } }); + await generate(); + const files = ["libs_generated.go", "embed_generated.go"].map(file => path.join(root, "tsc/internal/bundled", file)); + const timestamps = files.map(file => fs.statSync(file).mtimeMs); + assert.match((await generate()).stdout, /Bundled libraries are up to date\./); + assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); +}); + +test("bundled generation tracks libraries, validates inputs, and supports force", async context => { + const root = path.resolve(import.meta.dirname, "../.."); + const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_bundled-probe-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true, maxRetries: 10 })); + const notice = path.join(directory, "CopyrightNotice.txt"); + const libs = path.join(directory, "libs"); + fs.mkdirSync(libs); + fs.writeFileSync(notice, "Fixture notice\n\n"); + const library = path.join(libs, "lib.first.d.ts"); + const content = "Fixture notice\n\n\ndeclare const first: string;\n"; + fs.writeFileSync(library, content); + const command = [path.join(root, "tools/scripts/generateBundled.mts"), "--directory", directory]; + const generate = (...args: string[]) => x(process.execPath, [...command, ...args], { throwOnError: true, nodeOptions: { cwd: root } }); + await generate(); + const files = ["libs_generated.go", "embed_generated.go"].map(file => path.join(directory, file)); + const originals = files.map(file => fs.readFileSync(file)); + assert.match((await generate()).stdout, /Bundled libraries are up to date\./); + const added = path.join(libs, "lib.second.d.ts"); + fs.writeFileSync(added, content); + assert.match((await generate()).stdout, /Generated bundled libraries\./); + for (const file of files) assert.match(fs.readFileSync(file, "utf8"), /lib\.second\.d\.ts/); + fs.rmSync(added); + assert.match((await generate()).stdout, /Generated bundled libraries\./); + for (const [index, file] of files.entries()) { + assert.deepEqual(fs.readFileSync(file), originals[index]); + fs.writeFileSync(file, "modified"); + assert.match((await generate()).stdout, /Generated bundled libraries\./); + assert.deepEqual(fs.readFileSync(file), originals[index]); + fs.rmSync(file, { recursive: true, maxRetries: 10 }); + assert.match((await generate()).stdout, /Generated bundled libraries\./); + assert.deepEqual(fs.readFileSync(file), originals[index]); + } + fs.writeFileSync(library, content.replace("string", "number")); + assert.match((await generate()).stdout, /Generated bundled libraries\./); + fs.writeFileSync(library, content.replaceAll("\n", "\r\n")); + await assert.rejects(async () => await generate(), error => { + assert.match((error as { output: { stderr: string; }; }).output.stderr, /must use LF line endings/); + return true; + }); + fs.writeFileSync(library, content); + assert.match((await generate()).stdout, /Generated bundled libraries\./); + fs.mkdirSync(path.join(libs, "unexpected")); + await assert.rejects(async () => await generate(), error => { + assert.match((error as { output: { stderr: string; }; }).output.stderr, /unexpected entry/); + return true; + }); + fs.rmdirSync(path.join(libs, "unexpected")); + assert.match((await generate()).stdout, /Generated bundled libraries\./); + fs.writeFileSync(notice, "Changed fixture notice\n\n"); + await assert.rejects(async () => await generate(), error => { + assert.match((error as { output: { stderr: string; }; }).output.stderr, /must start with/); + return true; + }); + fs.writeFileSync(notice, "Fixture notice\n\n"); + assert.match((await generate()).stdout, /Generated bundled libraries\./); + assert.match((await generate("--force")).stdout, /Generated bundled libraries\./); + assert.match((await generate()).stdout, /Bundled libraries are up to date\./); + const forced = await x(process.execPath, command, { + throwOnError: true, + nodeOptions: { cwd: root, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, + }); + assert.match(forced.stdout, /Generated bundled libraries\./); + assert.match((await generate()).stdout, /Bundled libraries are up to date\./); +}); + +test("stringer generation tracks GOFILE, options, output, and force", async context => { + const root = path.resolve(import.meta.dirname, "../.."); + const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_stringer-probe-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const source = path.join(directory, "kind.go"); + const extra = path.join(directory, "extra.go"); + const output = path.join(directory, "kind_stringer_generated.go"); + fs.writeFileSync(source, "package probe\n\ntype Kind int\n\nconst KindFirst Kind = 0\n"); + const generate = (...args: string[]) => + x(process.execPath, [path.join(root, "tools/scripts/generateStringer.mts"), "-type=Kind", "-output=kind_stringer_generated.go", ...args], { + throwOnError: true, + nodeOptions: { cwd: directory, env: { ...process.env, GOFILE: "kind.go" } }, + }); + await generate(); + const timestamp = fs.statSync(output).mtimeMs; + assert.match((await generate()).stdout, /Stringer output is up to date\./); + assert.equal(fs.statSync(output).mtimeMs, timestamp); + fs.writeFileSync(extra, "package probe\n\nconst Unrelated = 1\n"); + assert.match((await generate()).stdout, /Stringer output is up to date\./); + fs.appendFileSync(source, "\nconst KindSecond Kind = 1\n"); + assert.match((await generate()).stdout, /Generated stringer output\./); + assert.match(fs.readFileSync(output, "utf8"), /KindSecond/); + fs.rmSync(extra); + assert.match((await generate()).stdout, /Stringer output is up to date\./); + fs.writeFileSync(source, "package probe\n\ntype Kind int\n\nconst KindFirst Kind = 0\n"); + assert.match((await generate()).stdout, /Generated stringer output\./); + assert.doesNotMatch(fs.readFileSync(output, "utf8"), /KindSecond/); + assert.match((await generate("-trimprefix=Kind")).stdout, /Generated stringer output\./); + assert.match(fs.readFileSync(output, "utf8"), /_Kind_name = "First"/); + assert.match((await generate("-trimprefix=Kind")).stdout, /Stringer output is up to date\./); + fs.appendFileSync(output, "\n"); + assert.match((await generate("-trimprefix=Kind")).stdout, /Generated stringer output\./); + fs.rmSync(output); + assert.match((await generate("-trimprefix=Kind")).stdout, /Generated stringer output\./); + assert.match((await generate("-trimprefix=Kind", "--force")).stdout, /Generated stringer output\./); + assert.match((await generate("-trimprefix=Kind")).stdout, /Stringer output is up to date\./); + const command = [path.join(root, "tools/scripts/generateStringer.mts"), "-type=Kind", "-output=kind_stringer_generated.go", "-trimprefix=Kind"]; + const explicit = await x(process.execPath, [...command, "--input", "kind.go"], { + throwOnError: true, + nodeOptions: { cwd: directory, env: { ...process.env, GOFILE: "" } }, + }); + assert.match(explicit.stdout, /Stringer output is up to date\./); + const forced = await x(process.execPath, command, { + throwOnError: true, + nodeOptions: { cwd: directory, env: { ...process.env, GOFILE: "kind.go", TSGO_HEREBY_FORCE: "1" } }, + }); + assert.match(forced.stdout, /Generated stringer output\./); + assert.match((await generate("-trimprefix=Kind")).stdout, /Stringer output is up to date\./); + fs.appendFileSync(source, "\nconst KindInvalid Kind = missing\n"); + await assert.rejects(async () => await generate("-trimprefix=Kind")); + fs.writeFileSync(source, "package probe\n\ntype Kind int\n\nconst KindFirst Kind = 0\n"); + assert.match((await generate("-trimprefix=Kind")).stdout, /Generated stringer output\./); + assert.match((await generate("-trimprefix=Kind")).stdout, /Stringer output is up to date\./); +}); + +test("moq generation tracks interface inputs, output, and force", async context => { + const root = path.resolve(import.meta.dirname, "../.."); + const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_moq-probe-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const source = path.join(directory, "client.go"); + const base = path.join(directory, "base.go"); + fs.mkdirSync(path.join(directory, "mock")); + const output = path.join(directory, "mock/mock_generated.go"); + fs.writeFileSync(base, "package probe\n\ntype Base interface { Read() string }\n"); + fs.writeFileSync(source, "package probe\n\ntype Client interface { Base }\n"); + const command = [path.join(root, "tools/scripts/generateMoq.mts"), "--input=client.go", "--input=base.go", "-stub", "-fmt=goimports", "-pkg=mock", "-out=mock/mock_generated.go"]; + const generate = (...args: string[]) => + x(process.execPath, [...command, ...args, ".", "Client"], { + throwOnError: true, + nodeOptions: { cwd: directory }, + }); + await generate(); + const timestamp = fs.statSync(output).mtimeMs; + assert.match((await generate()).stdout, /Moq output is up to date\./); + assert.equal(fs.statSync(output).mtimeMs, timestamp); + fs.writeFileSync(base, "package probe\n\ntype Base interface { Read() string; Close() error }\n"); + assert.match((await generate()).stdout, /Generated moq output\./); + assert.match(fs.readFileSync(output, "utf8"), /CloseFunc/); + fs.rmSync(output); + assert.match((await generate()).stdout, /Generated moq output\./); + assert.match((await generate("--force")).stdout, /Generated moq output\./); + assert.match((await generate()).stdout, /Moq output is up to date\./); + const forced = await x(process.execPath, [...command, ".", "Client"], { + throwOnError: true, + nodeOptions: { cwd: directory, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, + }); + assert.match(forced.stdout, /Generated moq output\./); + assert.match((await generate()).stdout, /Moq output is up to date\./); +}); + +test("diagnostic generation tracks Go and locale outputs and supports force", async context => { + const root = path.resolve(import.meta.dirname, "../.."); + const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_diagnostics-probe-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const generate = (...args: string[]) => + x(process.execPath, [path.join(root, "tools/scripts/generateDiagnostics.mts"), "--outDir", directory, ...args], { + throwOnError: true, + nodeOptions: { cwd: root }, + }); + await generate(); + const output = path.join(directory, "diagnostics_generated.go"); + const localized = path.join(directory, "loc_generated.go"); + const locales = fs.globSync("loc/*.json.gz", { cwd: directory }).map(file => path.join(directory, file)); + assert.ok(locales.length > 0); + const files = [output, localized, ...locales]; + const timestamps = files.map(file => fs.statSync(file).mtimeMs); + assert.match((await generate()).stdout, /Diagnostics are up to date\./); + assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); + const archive = fs.readFileSync(locales[0]); + fs.rmSync(locales[0]); + assert.match((await generate()).stdout, /Generated diagnostics\./); + assert.deepEqual(fs.readFileSync(locales[0]), archive); + fs.writeFileSync(locales[0], "modified"); + assert.match((await generate()).stdout, /Generated diagnostics\./); + assert.deepEqual(fs.readFileSync(locales[0]), archive); + const unexpected = path.join(directory, "loc/unexpected.json.gz"); + fs.writeFileSync(unexpected, "unexpected"); + assert.match((await generate()).stdout, /Generated diagnostics\./); + assert.equal(fs.existsSync(unexpected), false); + fs.rmSync(path.join(directory, "loc"), { recursive: true }); + assert.match((await generate()).stdout, /Generated diagnostics\./); + assert.deepEqual(fs.readFileSync(locales[0]), archive); + fs.rmSync(localized); + assert.match((await generate()).stdout, /Generated diagnostics\./); + assert.match((await generate("--force")).stdout, /Generated diagnostics\./); + assert.match((await generate()).stdout, /Diagnostics are up to date\./); + const forced = await x(process.execPath, [path.join(root, "tools/scripts/generateDiagnostics.mts"), "--outDir", directory], { + throwOnError: true, + nodeOptions: { cwd: root, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, + }); + assert.match(forced.stdout, /Generated diagnostics\./); + assert.match((await generate()).stdout, /Diagnostics are up to date\./); +}); + +test("Unicode generation skips unchanged tables and formatting", async () => { + const root = path.resolve(import.meta.dirname, "../.."); + const generate = () => x("go", ["-C", "./tsc", "generate", "./internal/stringutil"], { throwOnError: true, nodeOptions: { cwd: root } }); + await generate(); + const files = ["js_case_generated.go", "identifier_parts_generated.go"].map(file => path.join(root, "tsc/internal/stringutil", file)); + const timestamps = files.map(file => fs.statSync(file).mtimeMs); + const current = await generate(); + assert.match(current.stdout, /Unicode tables are up to date\./); + assert.doesNotMatch(current.stdout, /Formatted/); + assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); +}); + +test("Unicode generation repairs outputs and supports force", async context => { + const root = path.resolve(import.meta.dirname, "../.."); + const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_unicode-probe-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const command = [path.join(root, "tsc/internal/stringutil/_scripts/generate-unicode-data.mts"), "--outDir", directory]; + const generate = (...args: string[]) => x(process.execPath, [...command, ...args], { throwOnError: true, nodeOptions: { cwd: root } }); + await generate(); + const files = ["js_case_generated.go", "identifier_parts_generated.go"].map(file => path.join(directory, file)); + const originals = files.map(file => fs.readFileSync(file)); + const timestamps = files.map(file => fs.statSync(file).mtimeMs); + assert.match((await generate()).stdout, /Unicode tables are up to date\./); + assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); + for (const [index, file] of files.entries()) { + assert.deepEqual(originals[index], fs.readFileSync(path.join(root, "tsc/internal/stringutil", path.basename(file)))); + fs.writeFileSync(file, "modified"); + assert.match((await generate()).stdout, /Generated Unicode tables\./); + assert.deepEqual(fs.readFileSync(file), originals[index]); + fs.rmSync(file); + assert.match((await generate()).stdout, /Generated Unicode tables\./); + assert.deepEqual(fs.readFileSync(file), originals[index]); + } + const cacheFile = path.join(os.tmpdir(), "typescript-codegen", createHash("sha256").update(files[0]).digest("hex") + ".json"); + context.after(() => fs.rmSync(cacheFile, { force: true })); + fs.rmSync(cacheFile); + assert.match((await generate()).stdout, /Generated Unicode tables\./); + assert.match((await generate("--force")).stdout, /Generated Unicode tables\./); + assert.match((await generate()).stdout, /Unicode tables are up to date\./); + const forced = await x(process.execPath, command, { + throwOnError: true, + nodeOptions: { cwd: root, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, + }); + assert.match(forced.stdout, /Generated Unicode tables\./); + const overridden = await x(process.execPath, [...command, "--no-force"], { + throwOnError: true, + nodeOptions: { cwd: root, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, + }); + assert.match(overridden.stdout, /Unicode tables are up to date\./); +}); + +test("enum generation skips unchanged outputs and Go verification", async () => { + const root = path.resolve(import.meta.dirname, "../.."); + const generate = (force = false) => x("npx", ["hereby", "generate:enums", ...(force ? ["--force"] : [])], { throwOnError: true, nodeOptions: { cwd: root } }); + await generate(); + const files = [ + ...fs.globSync("packages/typescript/src/enums/*.ts", { cwd: root }), + "tsc/internal/api/enum_values_generated.go", + ].map(file => path.join(root, file)); + const timestamps = files.map(file => fs.statSync(file).mtimeMs); + const { stdout } = await generate(); + assert.match(stdout, /Enums are up to date\./); + assert.doesNotMatch(stdout, /Getting values from go/); + assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); + const forced = await generate(true); + assert.match(forced.stdout, /All generated values match Go\./); + assert.match((await generate()).stdout, /Enums are up to date\./); + const verifier = path.join(root, "tsc/internal/api/enum_values_generated.go"); + const cacheFile = path.join(os.tmpdir(), "typescript-codegen", createHash("sha256").update(verifier).digest("hex") + ".json"); + fs.rmSync(cacheFile); + const regenerated = await generate(); + assert.match(regenerated.stdout, /All generated values match Go\./); + assert.match((await generate()).stdout, /Enums are up to date\./); +}); + +test("AST generation forwards force to all schema generators", async () => { + const root = path.resolve(import.meta.dirname, "../.."); + const generate = (force = false) => x("npx", ["hereby", "generate:ast", ...(force ? ["--force"] : [])], { throwOnError: true, nodeOptions: { cwd: root } }); + await generate(); + const forced = await generate(true); + assert.match(forced.stdout, /Wrote .*encoder_generated\.go/); + assert.match(forced.stdout, /Wrote .*ast_generated\.go/); + assert.match(forced.stdout, /Generated .*ast\.generated\.ts/); + assert.doesNotMatch(forced.stdout, /\$ node .*tools\/scripts\/tsc\/generate\.ts/); + assert.doesNotMatch((await generate()).stdout, /(?:Wrote|Generated) /); +}); + +test("package generation forwards force to AST, encoder, and sync generators", async () => { + const root = path.resolve(import.meta.dirname, "../.."); + const generate = (force = false) => x("npm", ["run", "-w", "@typescript/typescript", "generate", ...(force ? ["--", "--force"] : [])], { throwOnError: true, nodeOptions: { cwd: root } }); + await generate(); + const forced = await generate(true); + assert.match(forced.stdout, /Generated .*ast\.generated\.ts/); + assert.match(forced.stdout, /Wrote .*encoder_generated\.go/); + assert.match(forced.stdout, /Formatting\.\.\./); + const current = await generate(); + assert.doesNotMatch(current.stdout, /(?:Wrote|Generated) |Formatting\.\.\./); + assert.match(current.stdout, /Sync API is up to date\./); +}); + +test("API protocol generation caches formatted output and supports force", async context => { + const root = path.resolve(import.meta.dirname, "../.."); + const output = path.join(root, "packages/typescript/src/api/proto.generated.ts"); + const generate = (force = false) => x("npx", ["hereby", "generate:api", ...(force ? ["--force"] : [])], { throwOnError: true, nodeOptions: { cwd: root } }); + await generate(); + const timestamp = fs.statSync(output).mtimeMs; + const current = await generate(); + assert.match(current.stdout, /API protocol is up to date\./); + assert.doesNotMatch(current.stdout, /\$ node .*generateAPI\.mts/); + assert.equal(fs.statSync(output).mtimeMs, timestamp); + const unrelated = path.join(root, "tsc/internal/parser/codegen_cache_probe.go"); + assert.equal(fs.existsSync(unrelated), false); + context.after(() => fs.rmSync(unrelated, { force: true })); + fs.writeFileSync(unrelated, "package parser\n"); + assert.match((await generate()).stdout, /API protocol is up to date\./); + fs.rmSync(unrelated); + const schema = path.join(root, "tsc/internal/api/requestfilesystem/codegen_cache_probe.go"); + assert.equal(fs.existsSync(schema), false); + const original = fs.readFileSync(output, "utf8"); + context.after(async () => { + if (fs.existsSync(schema)) { + fs.rmSync(schema); + await generate(true); + } + }); + fs.writeFileSync(schema, 'package requestfilesystem\n\nconst KindCodegenCacheProbe Kind = "codegen-cache-probe"\n'); + assert.match((await generate()).stdout, /Generated API protocol\./); + assert.match(fs.readFileSync(output, "utf8"), /"codegen-cache-probe"/); + fs.rmSync(schema); + assert.match((await generate()).stdout, /Generated API protocol\./); + assert.equal(fs.readFileSync(output, "utf8"), original); + assert.match((await generate(true)).stdout, /Generated API protocol\./); + assert.match((await generate()).stdout, /API protocol is up to date\./); + const direct = await x("node", ["./tools/scripts/generateAPI.mts"], { throwOnError: true, nodeOptions: { cwd: root } }); + assert.match(direct.stdout, /API protocol is up to date\./); + const nested = await x("go", ["-C", "./tsc", "generate", "./internal/api"], { + throwOnError: true, + nodeOptions: { cwd: root, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, + }); + assert.match(nested.stdout, /Generated API protocol\./); + assert.match((await generate()).stdout, /API protocol is up to date\./); +}); diff --git a/tools/scripts/tsc/generate-encoder.ts b/tools/scripts/tsc/generate-encoder.ts index b0c9f9e5f57ea..aa99a5d66d11e 100644 --- a/tools/scripts/tsc/generate-encoder.ts +++ b/tools/scripts/tsc/generate-encoder.ts @@ -10,10 +10,14 @@ * - packages/typescript/src/api/node/protocol.generated.ts */ -import * as fs from "node:fs"; import * as path from "node:path"; import { fileURLToPath } from "node:url"; -import { xSync } from "tinyexec"; +import { GeneratedFile } from "../gen/generatedFile.mts"; +import { + formatFilesSync, + parseGeneratorArgs, + repoRoot as ROOT, +} from "../gen/utils.mts"; import type { KindType, MemberInfo, @@ -21,8 +25,6 @@ import type { } from "./schema.ts"; import { api } from "./schema.ts"; -const ROOT = path.resolve(import.meta.dirname!, "../../.."); - // ═══════════════════════════════════════════════════════════════════════════ // Code writer // ═══════════════════════════════════════════════════════════════════════════ @@ -2009,14 +2011,13 @@ function emitRemoteNodeClassClose(w: CodeWriter) { // Main: generate and write all files // ═══════════════════════════════════════════════════════════════════════════ -function writeAndFormat(filePath: string, content: string, formatter: string) { - fs.writeFileSync(filePath, content); +function writeAndFormat(filePath: string, generate: () => string, force: boolean) { + const generated = new GeneratedFile(filePath, [import.meta.filename, path.join(ROOT, "tools/scripts/tsc/schema.ts"), path.join(ROOT, "tools/scripts/tsc/ast.json")]); + if (generated.isCurrent(force)) return; + generated.write(generate() + "\n"); try { - const [cmd, ...args] = formatter.split(" "); - xSync(cmd, [...args, filePath], { - throwOnError: true, - nodeOptions: { stdio: "inherit", cwd: ROOT }, - }); + formatFilesSync([filePath]); + generated.markCurrent(); } catch { console.warn(`Warning: formatter failed for ${filePath}`); @@ -2024,47 +2025,42 @@ function writeAndFormat(filePath: string, content: string, formatter: string) { console.log(`Wrote ${filePath}`); } -export default function main() { +export default function main(force = false) { console.log("Generating encoder/decoder code..."); - const goEncoder = generateGoEncoder(); writeAndFormat( path.join(ROOT, "tsc/internal/api/encoder/encoder_generated.go"), - goEncoder + "\n", - "dprint fmt", + generateGoEncoder, + force, ); - const goDecoder = generateGoDecoder(); writeAndFormat( path.join(ROOT, "tsc/internal/api/encoder/decoder_generated.go"), - goDecoder + "\n", - "dprint fmt", + generateGoDecoder, + force, ); - const tsProtocol = generateTSProtocol(); writeAndFormat( path.join(ROOT, "packages/typescript/src/api/node/protocol.generated.ts"), - tsProtocol + "\n", - "dprint fmt", + generateTSProtocol, + force, ); - const tsEncoder = generateTSEncoder(); writeAndFormat( path.join(ROOT, "packages/typescript/src/api/node/encoder.generated.ts"), - tsEncoder + "\n", - "dprint fmt", + generateTSEncoder, + force, ); - const tsNode = generateTSNodeGenerated(); writeAndFormat( path.join(ROOT, "packages/typescript/src/api/node/node.generated.ts"), - tsNode + "\n", - "dprint fmt", + generateTSNodeGenerated, + force, ); console.log("Done!"); } if (process.argv[1] === fileURLToPath(import.meta.url)) { - main(); + main(parseGeneratorArgs({}).force); } diff --git a/tools/scripts/tsc/generate-go-ast.ts b/tools/scripts/tsc/generate-go-ast.ts index e85658397729c..66235ca300424 100644 --- a/tools/scripts/tsc/generate-go-ast.ts +++ b/tools/scripts/tsc/generate-go-ast.ts @@ -14,10 +14,14 @@ * - Is*() type guard functions */ -import * as fs from "node:fs"; import * as path from "node:path"; import { fileURLToPath } from "node:url"; -import { xSync } from "tinyexec"; +import { GeneratedFile } from "../gen/generatedFile.mts"; +import { + formatFilesSync, + parseGeneratorArgs, + repoRoot as ROOT, +} from "../gen/utils.mts"; import type { MemberInfo, NodeType, @@ -32,8 +36,6 @@ import { // Load schema // ──────────────────────────────────────────────────────────────────────────── -const ROOT = path.resolve(import.meta.dirname!, "../../.."); - // Members that participate in factory/visitor/clone (excludes noFactory) function schemaMembers(node: NodeType): MemberInfo[] { return node.members.filter(m => !m.noFactory); @@ -984,8 +986,7 @@ function generateKind(): string { w.write(""); w.write("package ast"); w.write(""); - w.write("//go:generate go tool golang.org/x/tools/cmd/stringer -type=Kind -output=kind_stringer_generated.go"); - w.write("//go:generate npx dprint fmt kind_stringer_generated.go"); + w.write("//go:generate node ../../../tools/scripts/gen/generateStringer.mts -type=Kind -output=kind_stringer_generated.go"); w.write(""); w.write("type Kind int16"); w.write(""); @@ -1082,27 +1083,25 @@ function generateKind(): string { return w.toString(); } -function writeAndFormat(filePath: string, content: string) { - fs.writeFileSync(filePath, content); - xSync("dprint", ["fmt", filePath], { - throwOnError: true, - nodeOptions: { stdio: "inherit", cwd: ROOT }, - }); +function writeAndFormat(filePath: string, generateContent: () => string, force: boolean) { + const generated = new GeneratedFile(filePath, [import.meta.filename, path.join(ROOT, "tools/scripts/tsc/schema.ts"), path.join(ROOT, "tools/scripts/tsc/ast.json")]); + if (generated.isCurrent(force)) return; + generated.write(generateContent()); + formatFilesSync([filePath]); + generated.markCurrent(); console.log(`Wrote ${filePath}`); } -export default function main() { +export default function main(force = false) { console.log("Generating Go AST code..."); - const code = generate(); const outPath = path.join(ROOT, "tsc/internal/ast/ast_generated.go"); - writeAndFormat(outPath, code + "\n"); + writeAndFormat(outPath, () => generate() + "\n", force); - const kindCode = generateKind(); const kindOutPath = path.join(ROOT, "tsc/internal/ast/kind_generated.go"); - writeAndFormat(kindOutPath, kindCode + "\n"); + writeAndFormat(kindOutPath, () => generateKind() + "\n", force); } if (process.argv[1] === fileURLToPath(import.meta.url)) { - main(); + main(parseGeneratorArgs({}).force); } diff --git a/tools/scripts/tsc/generate-ts-ast.ts b/tools/scripts/tsc/generate-ts-ast.ts index 13f5bd3f1f088..bb82eeb80de24 100644 --- a/tools/scripts/tsc/generate-ts-ast.ts +++ b/tools/scripts/tsc/generate-ts-ast.ts @@ -8,10 +8,14 @@ * Usage: node tools/scripts/tsc/generate-ts-ast.ts */ -import * as fs from "node:fs"; import * as path from "node:path"; import { fileURLToPath } from "node:url"; -import { xSync } from "tinyexec"; +import { GeneratedFile } from "../gen/generatedFile.mts"; +import { + formatFilesSync, + parseGeneratorArgs, + repoRoot as ROOT, +} from "../gen/utils.mts"; import type { MemberInfo, NodeType, @@ -26,8 +30,6 @@ import { // Load schema // ──────────────────────────────────────────────────────────────────────────── -const ROOT = path.resolve(import.meta.dirname!, "../../.."); - // TS members only (filter noTS/inherited issues) function tsMembers(node: NodeType): MemberInfo[] { return node.members.filter(member => !member.isKindParam() && !member.noTS); @@ -1735,16 +1737,16 @@ function generateVisitor(): string { // Main // ──────────────────────────────────────────────────────────────────────────── -function writeAndFormat(filePath: string, content: string) { - fs.writeFileSync(filePath, content); - xSync("dprint", ["fmt", filePath], { - throwOnError: true, - nodeOptions: { stdio: "inherit", cwd: ROOT }, - }); +function writeAndFormat(filePath: string, generate: () => string, force: boolean) { + const generated = new GeneratedFile(filePath, [import.meta.filename, path.join(ROOT, "tools/scripts/tsc/schema.ts"), path.join(ROOT, "tools/scripts/tsc/ast.json")]); + if (generated.isCurrent(force)) return; + generated.write(generate()); + formatFilesSync([filePath]); + generated.markCurrent(); console.log(`Generated ${filePath}`); } -export default function main() { +export default function main(force = false) { console.log("Generating TS AST code..."); const factoryPath = path.join(ROOT, "packages/typescript/src/ast/factory.generated.ts"); @@ -1752,12 +1754,12 @@ export default function main() { const astGenPath = path.join(ROOT, "packages/typescript/src/ast/ast.generated.ts"); const visitorPath = path.join(ROOT, "packages/typescript/src/ast/visitor.generated.ts"); - writeAndFormat(astGenPath, generateAstGenerated()); - writeAndFormat(factoryPath, generateFactory()); - writeAndFormat(isGenPath, generateIsGenerated()); - writeAndFormat(visitorPath, generateVisitor()); + writeAndFormat(astGenPath, generateAstGenerated, force); + writeAndFormat(factoryPath, generateFactory, force); + writeAndFormat(isGenPath, generateIsGenerated, force); + writeAndFormat(visitorPath, generateVisitor, force); } if (process.argv[1] === fileURLToPath(import.meta.url)) { - main(); + main(parseGeneratorArgs({}).force); } diff --git a/tools/scripts/tsc/generate.ts b/tools/scripts/tsc/generate.ts index 6b1904cc4fb74..65f2a3997980a 100644 --- a/tools/scripts/tsc/generate.ts +++ b/tools/scripts/tsc/generate.ts @@ -1,14 +1,15 @@ import { fileURLToPath } from "node:url"; +import { parseGeneratorArgs } from "../gen/utils.mts"; import generateEncoder from "./generate-encoder.ts"; import generateGoAST from "./generate-go-ast.ts"; import generateTSAST from "./generate-ts-ast.ts"; -export default function generate() { - generateEncoder(); - generateGoAST(); - generateTSAST(); +export default function generate(force = false) { + generateEncoder(force); + generateGoAST(force); + generateTSAST(force); } if (process.argv[1] === fileURLToPath(import.meta.url)) { - generate(); + generate(parseGeneratorArgs({}).force); } diff --git a/tools/scripts/tsc/schema.ts b/tools/scripts/tsc/schema.ts index 299084c059ebf..6d2985d433435 100644 --- a/tools/scripts/tsc/schema.ts +++ b/tools/scripts/tsc/schema.ts @@ -2,6 +2,7 @@ import * as fs from "node:fs"; import * as path from "node:path"; +import { repoRoot as ROOT } from "../gen/utils.mts"; // ──────────────────────────────────────────────────────────────────────────── // Schema type definitions @@ -1260,7 +1261,6 @@ export class SchemaAPI { } } -const ROOT = path.resolve(import.meta.dirname!, "../../.."); export const api = new SchemaAPI(JSON.parse( fs.readFileSync(path.join(ROOT, "tools/scripts/tsc/ast.json"), "utf-8"), ) as Schema); diff --git a/tools/scripts/tsc/tsconfig.json b/tools/scripts/tsc/tsconfig.json index 170972c9a5557..48c845e97d604 100644 --- a/tools/scripts/tsc/tsconfig.json +++ b/tools/scripts/tsc/tsconfig.json @@ -1,11 +1,4 @@ -{ - "compilerOptions": { - "module": "nodenext", - "lib": ["esnext"], - "noEmit": true, - "verbatimModuleSyntax": true, - "erasableSyntaxOnly": true, - "allowImportingTsExtensions": true, - "types": ["node"] - } -} +{ + "extends": "../gen/tsconfig.json", + "include": ["**/*.ts"] +} diff --git a/tsc/internal/ast/kind_generated.go b/tsc/internal/ast/kind_generated.go index 8c67e7947572f..5875ad539a876 100644 --- a/tsc/internal/ast/kind_generated.go +++ b/tsc/internal/ast/kind_generated.go @@ -2,8 +2,7 @@ package ast -//go:generate go tool golang.org/x/tools/cmd/stringer -type=Kind -output=kind_stringer_generated.go -//go:generate npx dprint fmt kind_stringer_generated.go +//go:generate node ../../../tools/scripts/gen/generateStringer.mts -type=Kind -output=kind_stringer_generated.go type Kind int16 diff --git a/tsc/internal/bundled/bundled.go b/tsc/internal/bundled/bundled.go index f2121a0f7e1e9..b03040dc13e6c 100644 --- a/tsc/internal/bundled/bundled.go +++ b/tsc/internal/bundled/bundled.go @@ -11,7 +11,7 @@ import ( "github.com/microsoft/TypeScript/tsc/internal/vfs" ) -//go:generate go run generate.go +//go:generate node ../../../tools/scripts/gen/generateBundled.mts // Define the below here to consolidate documentation. diff --git a/tsc/internal/checker/types.go b/tsc/internal/checker/types.go index 1ec80b0a26f62..fc8b44f1ca980 100644 --- a/tsc/internal/checker/types.go +++ b/tsc/internal/checker/types.go @@ -11,8 +11,7 @@ import ( "github.com/microsoft/TypeScript/tsc/internal/evaluator" ) -//go:generate go tool golang.org/x/tools/cmd/stringer -type=SignatureKind -output=stringer_generated.go -//go:generate npx dprint fmt stringer_generated.go +//go:generate node ../../../tools/scripts/gen/generateStringer.mts -type=SignatureKind -output=stringer_generated.go // ParseFlags diff --git a/tsc/internal/core/compileroptions.go b/tsc/internal/core/compileroptions.go index 4e6e3ed01a008..70953dd418c72 100644 --- a/tsc/internal/core/compileroptions.go +++ b/tsc/internal/core/compileroptions.go @@ -9,9 +9,8 @@ import ( "github.com/microsoft/TypeScript/tsc/internal/tspath" ) -//go:generate go tool golang.org/x/tools/cmd/stringer -type=ModuleKind -trimprefix=ModuleKind -output=modulekind_stringer_generated.go -//go:generate go tool golang.org/x/tools/cmd/stringer -type=ScriptTarget -trimprefix=ScriptTarget -output=scripttarget_stringer_generated.go -//go:generate npx dprint fmt modulekind_stringer_generated.go scripttarget_stringer_generated.go +//go:generate node ../../../tools/scripts/gen/generateStringer.mts -type=ModuleKind -trimprefix=ModuleKind -output=modulekind_stringer_generated.go +//go:generate node ../../../tools/scripts/gen/generateStringer.mts -type=ScriptTarget -trimprefix=ScriptTarget -output=scripttarget_stringer_generated.go // CompilerOptions contains the compiler options exposed by the API. type CompilerOptions struct { diff --git a/tsc/internal/core/languagevariant.go b/tsc/internal/core/languagevariant.go index aacc5747b240c..49320bef85f24 100644 --- a/tsc/internal/core/languagevariant.go +++ b/tsc/internal/core/languagevariant.go @@ -1,7 +1,6 @@ package core -//go:generate go tool golang.org/x/tools/cmd/stringer -type=LanguageVariant -output=languagevariant_stringer_generated.go -//go:generate npx dprint fmt languagevariant_stringer_generated.go +//go:generate node ../../../tools/scripts/gen/generateStringer.mts -type=LanguageVariant -output=languagevariant_stringer_generated.go type LanguageVariant int32 diff --git a/tsc/internal/core/scriptkind.go b/tsc/internal/core/scriptkind.go index 95bd8abd82b28..0f04a901d45a4 100644 --- a/tsc/internal/core/scriptkind.go +++ b/tsc/internal/core/scriptkind.go @@ -1,7 +1,6 @@ package core -//go:generate go tool golang.org/x/tools/cmd/stringer -type=ScriptKind -output=scriptkind_stringer_generated.go -//go:generate npx dprint fmt scriptkind_stringer_generated.go +//go:generate node ../../../tools/scripts/gen/generateStringer.mts -type=ScriptKind -output=scriptkind_stringer_generated.go type ScriptKind int32 diff --git a/tsc/internal/core/tristate.go b/tsc/internal/core/tristate.go index 8745040b9ab2e..005ecfe086504 100644 --- a/tsc/internal/core/tristate.go +++ b/tsc/internal/core/tristate.go @@ -1,7 +1,6 @@ package core -//go:generate go tool golang.org/x/tools/cmd/stringer -type=Tristate -output=tristate_stringer_generated.go -//go:generate npx dprint fmt tristate_stringer_generated.go +//go:generate node ../../../tools/scripts/gen/generateStringer.mts -type=Tristate -output=tristate_stringer_generated.go // Tristate diff --git a/tsc/internal/diagnostics/diagnostics.go b/tsc/internal/diagnostics/diagnostics.go index 294554b2a2de7..3b2be93991d2a 100644 --- a/tsc/internal/diagnostics/diagnostics.go +++ b/tsc/internal/diagnostics/diagnostics.go @@ -13,9 +13,8 @@ import ( "golang.org/x/text/language" ) -//go:generate go run generate.go -diagnostics ./diagnostics_generated.go -loc ./loc_generated.go -locdir ./loc -//go:generate go tool golang.org/x/tools/cmd/stringer -type=Category -output=stringer_generated.go -//go:generate npx dprint fmt diagnostics_generated.go loc_generated.go stringer_generated.go +//go:generate node ../../../tools/scripts/gen/generateDiagnostics.mts +//go:generate node ../../../tools/scripts/gen/generateStringer.mts -type=Category -output=stringer_generated.go type Category int32 diff --git a/tsc/internal/ls/autoimport/export.go b/tsc/internal/ls/autoimport/export.go index 468f5ae35776d..294136e7b48f9 100644 --- a/tsc/internal/ls/autoimport/export.go +++ b/tsc/internal/ls/autoimport/export.go @@ -8,8 +8,7 @@ import ( "github.com/microsoft/TypeScript/tsc/internal/tspath" ) -//go:generate go tool golang.org/x/tools/cmd/stringer -type=ExportSyntax -output=export_stringer_generated.go -//go:generate npx dprint fmt export_stringer_generated.go +//go:generate node ../../../../tools/scripts/gen/generateStringer.mts -type=ExportSyntax -output=export_stringer_generated.go // ModuleID uniquely identifies a module across multiple declarations. // If the export is from an ambient module declaration, this is the module name. diff --git a/tsc/internal/project/project.go b/tsc/internal/project/project.go index 354e778da94be..ea3a8fcc35740 100644 --- a/tsc/internal/project/project.go +++ b/tsc/internal/project/project.go @@ -40,8 +40,7 @@ func SyntheticProgramID(path tspath.Path) (int, bool) { return id, err == nil && id > 0 } -//go:generate go tool golang.org/x/tools/cmd/stringer -type=Kind -trimprefix=Kind -output=project_stringer_generated.go -//go:generate npx dprint fmt project_stringer_generated.go +//go:generate node ../../../tools/scripts/gen/generateStringer.mts -type=Kind -trimprefix=Kind -output=project_stringer_generated.go type Kind int diff --git a/tsc/internal/stringutil/_scripts/generate-unicode-data.mts b/tsc/internal/stringutil/_scripts/generate-unicode-data.mts index 0353b2141aae1..e24c8a5d7152b 100644 --- a/tsc/internal/stringutil/_scripts/generate-unicode-data.mts +++ b/tsc/internal/stringutil/_scripts/generate-unicode-data.mts @@ -1,7 +1,12 @@ #!/usr/bin/env node -import * as fs from "fs"; +import { fileURLToPath } from "node:url"; import * as path from "path"; +import { GeneratedFile } from "../../../../tools/scripts/gen/generatedFile.mts"; +import { + formatFiles, + parseGeneratorArgs, +} from "../../../../tools/scripts/gen/utils.mts"; // All Unicode data is sourced from a single version-pinned @unicode/unicode-* // package so the generated tables are reproducible regardless of the Node.js @@ -12,8 +17,6 @@ const UNICODE_VERSION = "15.1.0"; const PACKAGE = `@unicode/unicode-${UNICODE_VERSION}`; const scriptDir = import.meta.dirname; -const CASE_OUTPUT_PATH = path.join(scriptDir, "..", "js_case_generated.go"); -const IDENTIFIER_OUTPUT_PATH = path.join(scriptDir, "..", "identifier_parts_generated.go"); // A *unicode.RangeTable is split into 16-bit (BMP) and 32-bit (astral) ranges, // each carrying a stride so arithmetic sequences (e.g. the alternating @@ -228,20 +231,40 @@ ${renderRangeTable("unicodeESNextIdentifierPart", partTable)} `; } -async function main() { +export default async function generateUnicode(force = false, outDir = path.resolve(scriptDir, "..")) { + outDir = path.resolve(outDir); + const inputs = [ + import.meta.filename, + fileURLToPath(import.meta.resolve(`${PACKAGE}/package.json`)), + ]; + const caseOutput = new GeneratedFile(path.join(outDir, "js_case_generated.go"), inputs); + const identifierOutput = new GeneratedFile(path.join(outDir, "identifier_parts_generated.go"), inputs); + const generatedFiles = [caseOutput, identifierOutput]; + if (generatedFiles.every(file => file.isCurrent(force))) { + console.log("Unicode tables are up to date."); + return; + } + + for (const file of generatedFiles) file.invalidate(); const simpleLowercase = await loadSimpleMapping("Simple_Case_Mapping/Lowercase"); const simpleUppercase = await loadSimpleMapping("Simple_Case_Mapping/Uppercase"); const entries = await buildSpecialCasing(simpleLowercase, simpleUppercase); const casedTable = toRangeTable(await loadCodePoints("Binary_Property/Cased")); const caseIgnorableTable = toRangeTable(await loadCodePoints("Binary_Property/Case_Ignorable")); - fs.writeFileSync(CASE_OUTPUT_PATH, renderCaseFile(entries, casedTable, caseIgnorableTable)); + caseOutput.write(renderCaseFile(entries, casedTable, caseIgnorableTable)); const idStart = await loadCodePoints("Binary_Property/ID_Start"); const idContinue = await loadCodePoints("Binary_Property/ID_Continue"); // Other_ID_Start/Other_ID_Continue are already folded into ID_Start/ID_Continue. const startTable = toRangeTable(idStart); const partTable = toRangeTable([...idContinue, ...idStart]); - fs.writeFileSync(IDENTIFIER_OUTPUT_PATH, renderIdentifierFile(startTable, partTable)); + identifierOutput.write(renderIdentifierFile(startTable, partTable)); + await formatFiles(generatedFiles.map(file => file.fileName)); + for (const file of generatedFiles) file.markCurrent(); + console.log("Generated Unicode tables."); } -await main(); +if (process.argv[1] === import.meta.filename) { + const { values, force } = parseGeneratorArgs({ outDir: { type: "string" } }); + await generateUnicode(force, values.outDir); +} diff --git a/tsc/internal/stringutil/generate.go b/tsc/internal/stringutil/generate.go index 196e2d59e3904..408eb756548cf 100644 --- a/tsc/internal/stringutil/generate.go +++ b/tsc/internal/stringutil/generate.go @@ -1,4 +1,3 @@ package stringutil //go:generate node ./_scripts/generate-unicode-data.mts -//go:generate npx dprint fmt js_case_generated.go identifier_parts_generated.go diff --git a/tsc/internal/testutil/projecttestutil/projecttestutil.go b/tsc/internal/testutil/projecttestutil/projecttestutil.go index ca1af7fb7edae..bcd6f02042966 100644 --- a/tsc/internal/testutil/projecttestutil/projecttestutil.go +++ b/tsc/internal/testutil/projecttestutil/projecttestutil.go @@ -23,11 +23,9 @@ import ( "github.com/microsoft/TypeScript/tsc/internal/vfs/vfstest" ) -//go:generate go tool github.com/matryer/moq -stub -fmt goimports -pkg projecttestutil -out clientmock_generated.go ../../project Client -//go:generate npx dprint fmt clientmock_generated.go +//go:generate node ../../../../tools/scripts/gen/generateMoq.mts --input=../../project/client.go -stub -fmt goimports -pkg projecttestutil -out clientmock_generated.go ../../project Client -//go:generate go tool github.com/matryer/moq -stub -fmt goimports -pkg projecttestutil -out npmexecutormock_generated.go ../../project/ata NpmExecutor -//go:generate npx dprint fmt npmexecutormock_generated.go +//go:generate node ../../../../tools/scripts/gen/generateMoq.mts --input=../../project/ata/ata.go -stub -fmt goimports -pkg projecttestutil -out npmexecutormock_generated.go ../../project/ata NpmExecutor const ( TestTypingsLocation = "/home/src/Library/Caches/typescript" diff --git a/tsc/internal/vfs/vfs.go b/tsc/internal/vfs/vfs.go index f3126c7eb50a5..2a7000cb7c828 100644 --- a/tsc/internal/vfs/vfs.go +++ b/tsc/internal/vfs/vfs.go @@ -5,8 +5,7 @@ import ( "time" ) -//go:generate go tool github.com/matryer/moq -fmt goimports -out vfsmock/mock_generated.go -pkg vfsmock . FS -//go:generate npx dprint fmt vfsmock/mock_generated.go +//go:generate node ../../../tools/scripts/gen/generateMoq.mts --input=vfs.go -fmt goimports -out vfsmock/mock_generated.go -pkg vfsmock . FS // FS is a file system abstraction. type FS interface { diff --git a/tsc/internal/vfs/vfsmatch/vfsmatch.go b/tsc/internal/vfs/vfsmatch/vfsmatch.go index 4396f93831b4d..6abe2fcec0133 100644 --- a/tsc/internal/vfs/vfsmatch/vfsmatch.go +++ b/tsc/internal/vfs/vfsmatch/vfsmatch.go @@ -12,8 +12,7 @@ import ( "github.com/microsoft/TypeScript/tsc/internal/vfs" ) -//go:generate go tool golang.org/x/tools/cmd/stringer -type=Usage -trimprefix=Usage -output=stringer_generated.go -//go:generate npx dprint fmt stringer_generated.go +//go:generate node ../../../../tools/scripts/gen/generateStringer.mts -type=Usage -trimprefix=Usage -output=stringer_generated.go // This file implements the glob matching algorithm specified in MATCHING_ALGORITHM.md. From a9819a9f068c87191877a939b0430980a97a7636 Mon Sep 17 00:00:00 2001 From: Wesley Wigham Date: Fri, 18 Sep 2026 18:25:37 -0700 Subject: [PATCH 2/6] Man, I thought I removed these ages ago --- tools/scripts/generatedFile.mts | 77 ----- tools/scripts/generatedFile.test.mts | 488 --------------------------- 2 files changed, 565 deletions(-) delete mode 100644 tools/scripts/generatedFile.mts delete mode 100644 tools/scripts/generatedFile.test.mts diff --git a/tools/scripts/generatedFile.mts b/tools/scripts/generatedFile.mts deleted file mode 100644 index d05521edc7c7c..0000000000000 --- a/tools/scripts/generatedFile.mts +++ /dev/null @@ -1,77 +0,0 @@ -import { createHash } from "node:crypto"; -import * as fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -const repoRoot = path.resolve(import.meta.dirname, "../.."); -const commonInputs = [ - import.meta.filename, - path.join(repoRoot, "package-lock.json"), - path.join(repoRoot, ".dprint.jsonc"), -]; - -function digest(value: string | Buffer): string { - return createHash("sha256").update(value).digest("hex"); -} - -function readIfExists(file: string): Buffer | undefined { - try { - return fs.readFileSync(file); - } - catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - } -} - -export class GeneratedFile { - readonly fileName: string; - private readonly inputs: readonly string[]; - private readonly inputHash: string; - private readonly cacheFile: string; - private readonly key: unknown; - - constructor(fileName: string, inputs: readonly string[], cacheDirectory = path.join(os.tmpdir(), "typescript-codegen"), key?: unknown) { - this.fileName = path.resolve(fileName); - this.inputs = [...new Set([...commonInputs, ...inputs].map(file => path.resolve(file)))].sort(); - this.key = key; - this.inputHash = this.hashInputs(); - this.cacheFile = path.join(cacheDirectory, digest(this.fileName) + ".json"); - } - - private hashInputs(): string { - return digest(JSON.stringify([ - process.version, - process.platform, - process.arch, - this.key, - this.inputs.map(file => [file, digest(fs.readFileSync(file))]), - ])); - } - - private state(output: Buffer): string { - return JSON.stringify([this.inputHash, digest(output)]); - } - - isCurrent(force = false): boolean { - if (force) return false; - const output = readIfExists(this.fileName); - return output !== undefined && readIfExists(this.cacheFile)?.toString() === this.state(output); - } - - invalidate(): void { - fs.rmSync(this.cacheFile, { force: true }); - } - - write(content: string): void { - this.invalidate(); - fs.mkdirSync(path.dirname(this.fileName), { recursive: true }); - fs.writeFileSync(this.fileName, content); - } - - markCurrent(): void { - if (this.inputHash !== this.hashInputs()) return; - const state = this.state(fs.readFileSync(this.fileName)); - fs.mkdirSync(path.dirname(this.cacheFile), { recursive: true }); - fs.writeFileSync(this.cacheFile, state); - } -} diff --git a/tools/scripts/generatedFile.test.mts b/tools/scripts/generatedFile.test.mts deleted file mode 100644 index 2f510a6e2aa32..0000000000000 --- a/tools/scripts/generatedFile.test.mts +++ /dev/null @@ -1,488 +0,0 @@ -import assert from "node:assert/strict"; -import { createHash } from "node:crypto"; -import * as fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { test } from "node:test"; -import { x } from "tinyexec"; -import { GeneratedFile } from "./generatedFile.mts"; - -test("generator options preserve tool arguments and force overrides", async () => { - const { parseGeneratorArgs, resolveForce } = await import("./gen/utils.mts"); - const options = { type: { type: "string" }, input: { type: "string", multiple: true } } as const; - const args = ["-type", "Kind", "--input=kind.go", "--input", "extra.go", "--force", ".", "Kind:Mock"]; - const parsed = parseGeneratorArgs(options, args, true); - assert.equal(parsed.values.type, "Kind"); - assert.deepEqual(parsed.values.input, ["kind.go", "extra.go"]); - assert.deepEqual(parsed.positionals, [".", "Kind:Mock"]); - assert.deepEqual(parsed.toolArgs, ["-type", "Kind", ".", "Kind:Mock"]); - assert.equal(parsed.force, true); - assert.equal(resolveForce(undefined, "ON"), true); - assert.equal(resolveForce(undefined, "0"), false); - assert.equal(resolveForce(false, "1"), false); - assert.equal(parseGeneratorArgs({}, ["--no-force"]).force, false); -}); - -test("generated files are current only while their inputs and formatted output match", context => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-codegen-")); - context.after(() => fs.rmSync(directory, { recursive: true, force: true })); - const input = path.join(directory, "input.ts"); - const output = path.join(directory, "output.ts"); - const cache = path.join(directory, "cache"); - fs.writeFileSync(input, "input"); - const generated = () => new GeneratedFile(output, [input], cache); - let file = generated(); - assert.equal(file.isCurrent(), false); - file.write("unformatted"); - fs.writeFileSync(output, "formatted"); - file.markCurrent(); - assert.equal(generated().isCurrent(), true); - assert.equal(generated().isCurrent(true), false); - file.invalidate(); - assert.equal(generated().isCurrent(), false); - file.write("formatted"); - file.markCurrent(); - assert.equal(generated().isCurrent(), true); - fs.utimesSync(input, new Date(), new Date()); - assert.equal(generated().isCurrent(), true); - fs.writeFileSync(input, "changed"); - assert.equal(generated().isCurrent(), false); - fs.writeFileSync(input, "input"); - assert.equal(generated().isCurrent(), true); - fs.writeFileSync(output, "edited"); - assert.equal(generated().isCurrent(), false); - file = generated(); - file.write("formatted"); - assert.equal(generated().isCurrent(), false); - file.markCurrent(); - assert.equal(generated().isCurrent(), true); - const dependency = path.join(directory, "generator.ts"); - fs.writeFileSync(dependency, "generator"); - assert.equal(new GeneratedFile(output, [input, dependency], cache).isCurrent(), false); - fs.rmSync(dependency); - assert.throws(() => new GeneratedFile(output, [input, dependency], cache), { code: "ENOENT" }); - fs.rmSync(output); - assert.equal(generated().isCurrent(), false); -}); - -test("changing inputs during generation does not mark stale output current", context => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-codegen-")); - context.after(() => fs.rmSync(directory, { recursive: true, force: true })); - const input = path.join(directory, "input.ts"); - const output = path.join(directory, "output.ts"); - const cache = path.join(directory, "cache"); - fs.writeFileSync(input, "before"); - const file = new GeneratedFile(output, [input], cache); - file.write("generated"); - fs.writeFileSync(input, "after"); - file.markCurrent(); - assert.equal(new GeneratedFile(output, [input], cache).isCurrent(), false); -}); - -test("default metadata has a stable path in the OS temporary directory", context => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-codegen-")); - context.after(() => fs.rmSync(directory, { recursive: true, force: true })); - const output = path.join(directory, "output.ts"); - const cacheFile = path.join(os.tmpdir(), "typescript-codegen", createHash("sha256").update(output).digest("hex") + ".json"); - context.after(() => fs.rmSync(cacheFile, { force: true })); - const file = new GeneratedFile(output, []); - file.write("generated"); - file.markCurrent(); - assert.equal(fs.existsSync(cacheFile), true); - assert.equal(new GeneratedFile(output, []).isCurrent(), true); - fs.rmSync(cacheFile); - assert.equal(new GeneratedFile(output, []).isCurrent(), false); - file.markCurrent(); - assert.equal(new GeneratedFile(output, []).isCurrent(), true); -}); - -test("metadata directories can disappear during generation", context => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-codegen-")); - context.after(() => fs.rmSync(directory, { recursive: true, force: true })); - const output = path.join(directory, "output.ts"); - const cache = path.join(directory, "cache"); - const file = new GeneratedFile(output, [], cache); - file.write("generated"); - file.markCurrent(); - fs.rmSync(cache, { recursive: true }); - assert.equal(file.isCurrent(), false); - file.write("regenerated"); - file.markCurrent(); - assert.equal(new GeneratedFile(output, [], cache).isCurrent(), true); -}); - -test("generated files track each output independently", context => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-codegen-")); - context.after(() => fs.rmSync(directory, { recursive: true, force: true })); - const files = ["output.go", "locale.gz"].map(file => new GeneratedFile(path.join(directory, file), [], path.join(directory, "cache"))); - for (const file of files) { - file.write("generated"); - file.markCurrent(); - } - assert.equal(files.every(file => file.isCurrent()), true); - fs.writeFileSync(files[1].fileName, "modified"); - assert.equal(files[0].isCurrent(), true); - assert.equal(files[1].isCurrent(), false); - fs.writeFileSync(files[1].fileName, "generated"); - assert.equal(files.every(file => file.isCurrent()), true); - fs.rmSync(files[1].fileName); - assert.equal(files[0].isCurrent(), true); - assert.equal(files[1].isCurrent(), false); - files[1].write("generated"); - files[1].markCurrent(); - files[1].invalidate(); - assert.equal(files[0].isCurrent(), true); - assert.equal(files[1].isCurrent(), false); -}); - -test("bundled generation skips unchanged library outputs", async () => { - const root = path.resolve(import.meta.dirname, "../.."); - const generate = () => x("go", ["-C", "./tsc", "generate", "./internal/bundled"], { throwOnError: true, nodeOptions: { cwd: root } }); - await generate(); - const files = ["libs_generated.go", "embed_generated.go"].map(file => path.join(root, "tsc/internal/bundled", file)); - const timestamps = files.map(file => fs.statSync(file).mtimeMs); - assert.match((await generate()).stdout, /Bundled libraries are up to date\./); - assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); -}); - -test("bundled generation tracks libraries, validates inputs, and supports force", async context => { - const root = path.resolve(import.meta.dirname, "../.."); - const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_bundled-probe-")); - context.after(() => fs.rmSync(directory, { recursive: true, force: true, maxRetries: 10 })); - const notice = path.join(directory, "CopyrightNotice.txt"); - const libs = path.join(directory, "libs"); - fs.mkdirSync(libs); - fs.writeFileSync(notice, "Fixture notice\n\n"); - const library = path.join(libs, "lib.first.d.ts"); - const content = "Fixture notice\n\n\ndeclare const first: string;\n"; - fs.writeFileSync(library, content); - const command = [path.join(root, "tools/scripts/generateBundled.mts"), "--directory", directory]; - const generate = (...args: string[]) => x(process.execPath, [...command, ...args], { throwOnError: true, nodeOptions: { cwd: root } }); - await generate(); - const files = ["libs_generated.go", "embed_generated.go"].map(file => path.join(directory, file)); - const originals = files.map(file => fs.readFileSync(file)); - assert.match((await generate()).stdout, /Bundled libraries are up to date\./); - const added = path.join(libs, "lib.second.d.ts"); - fs.writeFileSync(added, content); - assert.match((await generate()).stdout, /Generated bundled libraries\./); - for (const file of files) assert.match(fs.readFileSync(file, "utf8"), /lib\.second\.d\.ts/); - fs.rmSync(added); - assert.match((await generate()).stdout, /Generated bundled libraries\./); - for (const [index, file] of files.entries()) { - assert.deepEqual(fs.readFileSync(file), originals[index]); - fs.writeFileSync(file, "modified"); - assert.match((await generate()).stdout, /Generated bundled libraries\./); - assert.deepEqual(fs.readFileSync(file), originals[index]); - fs.rmSync(file, { recursive: true, maxRetries: 10 }); - assert.match((await generate()).stdout, /Generated bundled libraries\./); - assert.deepEqual(fs.readFileSync(file), originals[index]); - } - fs.writeFileSync(library, content.replace("string", "number")); - assert.match((await generate()).stdout, /Generated bundled libraries\./); - fs.writeFileSync(library, content.replaceAll("\n", "\r\n")); - await assert.rejects(async () => await generate(), error => { - assert.match((error as { output: { stderr: string; }; }).output.stderr, /must use LF line endings/); - return true; - }); - fs.writeFileSync(library, content); - assert.match((await generate()).stdout, /Generated bundled libraries\./); - fs.mkdirSync(path.join(libs, "unexpected")); - await assert.rejects(async () => await generate(), error => { - assert.match((error as { output: { stderr: string; }; }).output.stderr, /unexpected entry/); - return true; - }); - fs.rmdirSync(path.join(libs, "unexpected")); - assert.match((await generate()).stdout, /Generated bundled libraries\./); - fs.writeFileSync(notice, "Changed fixture notice\n\n"); - await assert.rejects(async () => await generate(), error => { - assert.match((error as { output: { stderr: string; }; }).output.stderr, /must start with/); - return true; - }); - fs.writeFileSync(notice, "Fixture notice\n\n"); - assert.match((await generate()).stdout, /Generated bundled libraries\./); - assert.match((await generate("--force")).stdout, /Generated bundled libraries\./); - assert.match((await generate()).stdout, /Bundled libraries are up to date\./); - const forced = await x(process.execPath, command, { - throwOnError: true, - nodeOptions: { cwd: root, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, - }); - assert.match(forced.stdout, /Generated bundled libraries\./); - assert.match((await generate()).stdout, /Bundled libraries are up to date\./); -}); - -test("stringer generation tracks GOFILE, options, output, and force", async context => { - const root = path.resolve(import.meta.dirname, "../.."); - const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_stringer-probe-")); - context.after(() => fs.rmSync(directory, { recursive: true, force: true })); - const source = path.join(directory, "kind.go"); - const extra = path.join(directory, "extra.go"); - const output = path.join(directory, "kind_stringer_generated.go"); - fs.writeFileSync(source, "package probe\n\ntype Kind int\n\nconst KindFirst Kind = 0\n"); - const generate = (...args: string[]) => - x(process.execPath, [path.join(root, "tools/scripts/generateStringer.mts"), "-type=Kind", "-output=kind_stringer_generated.go", ...args], { - throwOnError: true, - nodeOptions: { cwd: directory, env: { ...process.env, GOFILE: "kind.go" } }, - }); - await generate(); - const timestamp = fs.statSync(output).mtimeMs; - assert.match((await generate()).stdout, /Stringer output is up to date\./); - assert.equal(fs.statSync(output).mtimeMs, timestamp); - fs.writeFileSync(extra, "package probe\n\nconst Unrelated = 1\n"); - assert.match((await generate()).stdout, /Stringer output is up to date\./); - fs.appendFileSync(source, "\nconst KindSecond Kind = 1\n"); - assert.match((await generate()).stdout, /Generated stringer output\./); - assert.match(fs.readFileSync(output, "utf8"), /KindSecond/); - fs.rmSync(extra); - assert.match((await generate()).stdout, /Stringer output is up to date\./); - fs.writeFileSync(source, "package probe\n\ntype Kind int\n\nconst KindFirst Kind = 0\n"); - assert.match((await generate()).stdout, /Generated stringer output\./); - assert.doesNotMatch(fs.readFileSync(output, "utf8"), /KindSecond/); - assert.match((await generate("-trimprefix=Kind")).stdout, /Generated stringer output\./); - assert.match(fs.readFileSync(output, "utf8"), /_Kind_name = "First"/); - assert.match((await generate("-trimprefix=Kind")).stdout, /Stringer output is up to date\./); - fs.appendFileSync(output, "\n"); - assert.match((await generate("-trimprefix=Kind")).stdout, /Generated stringer output\./); - fs.rmSync(output); - assert.match((await generate("-trimprefix=Kind")).stdout, /Generated stringer output\./); - assert.match((await generate("-trimprefix=Kind", "--force")).stdout, /Generated stringer output\./); - assert.match((await generate("-trimprefix=Kind")).stdout, /Stringer output is up to date\./); - const command = [path.join(root, "tools/scripts/generateStringer.mts"), "-type=Kind", "-output=kind_stringer_generated.go", "-trimprefix=Kind"]; - const explicit = await x(process.execPath, [...command, "--input", "kind.go"], { - throwOnError: true, - nodeOptions: { cwd: directory, env: { ...process.env, GOFILE: "" } }, - }); - assert.match(explicit.stdout, /Stringer output is up to date\./); - const forced = await x(process.execPath, command, { - throwOnError: true, - nodeOptions: { cwd: directory, env: { ...process.env, GOFILE: "kind.go", TSGO_HEREBY_FORCE: "1" } }, - }); - assert.match(forced.stdout, /Generated stringer output\./); - assert.match((await generate("-trimprefix=Kind")).stdout, /Stringer output is up to date\./); - fs.appendFileSync(source, "\nconst KindInvalid Kind = missing\n"); - await assert.rejects(async () => await generate("-trimprefix=Kind")); - fs.writeFileSync(source, "package probe\n\ntype Kind int\n\nconst KindFirst Kind = 0\n"); - assert.match((await generate("-trimprefix=Kind")).stdout, /Generated stringer output\./); - assert.match((await generate("-trimprefix=Kind")).stdout, /Stringer output is up to date\./); -}); - -test("moq generation tracks interface inputs, output, and force", async context => { - const root = path.resolve(import.meta.dirname, "../.."); - const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_moq-probe-")); - context.after(() => fs.rmSync(directory, { recursive: true, force: true })); - const source = path.join(directory, "client.go"); - const base = path.join(directory, "base.go"); - fs.mkdirSync(path.join(directory, "mock")); - const output = path.join(directory, "mock/mock_generated.go"); - fs.writeFileSync(base, "package probe\n\ntype Base interface { Read() string }\n"); - fs.writeFileSync(source, "package probe\n\ntype Client interface { Base }\n"); - const command = [path.join(root, "tools/scripts/generateMoq.mts"), "--input=client.go", "--input=base.go", "-stub", "-fmt=goimports", "-pkg=mock", "-out=mock/mock_generated.go"]; - const generate = (...args: string[]) => - x(process.execPath, [...command, ...args, ".", "Client"], { - throwOnError: true, - nodeOptions: { cwd: directory }, - }); - await generate(); - const timestamp = fs.statSync(output).mtimeMs; - assert.match((await generate()).stdout, /Moq output is up to date\./); - assert.equal(fs.statSync(output).mtimeMs, timestamp); - fs.writeFileSync(base, "package probe\n\ntype Base interface { Read() string; Close() error }\n"); - assert.match((await generate()).stdout, /Generated moq output\./); - assert.match(fs.readFileSync(output, "utf8"), /CloseFunc/); - fs.rmSync(output); - assert.match((await generate()).stdout, /Generated moq output\./); - assert.match((await generate("--force")).stdout, /Generated moq output\./); - assert.match((await generate()).stdout, /Moq output is up to date\./); - const forced = await x(process.execPath, [...command, ".", "Client"], { - throwOnError: true, - nodeOptions: { cwd: directory, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, - }); - assert.match(forced.stdout, /Generated moq output\./); - assert.match((await generate()).stdout, /Moq output is up to date\./); -}); - -test("diagnostic generation tracks Go and locale outputs and supports force", async context => { - const root = path.resolve(import.meta.dirname, "../.."); - const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_diagnostics-probe-")); - context.after(() => fs.rmSync(directory, { recursive: true, force: true })); - const generate = (...args: string[]) => - x(process.execPath, [path.join(root, "tools/scripts/generateDiagnostics.mts"), "--outDir", directory, ...args], { - throwOnError: true, - nodeOptions: { cwd: root }, - }); - await generate(); - const output = path.join(directory, "diagnostics_generated.go"); - const localized = path.join(directory, "loc_generated.go"); - const locales = fs.globSync("loc/*.json.gz", { cwd: directory }).map(file => path.join(directory, file)); - assert.ok(locales.length > 0); - const files = [output, localized, ...locales]; - const timestamps = files.map(file => fs.statSync(file).mtimeMs); - assert.match((await generate()).stdout, /Diagnostics are up to date\./); - assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); - const archive = fs.readFileSync(locales[0]); - fs.rmSync(locales[0]); - assert.match((await generate()).stdout, /Generated diagnostics\./); - assert.deepEqual(fs.readFileSync(locales[0]), archive); - fs.writeFileSync(locales[0], "modified"); - assert.match((await generate()).stdout, /Generated diagnostics\./); - assert.deepEqual(fs.readFileSync(locales[0]), archive); - const unexpected = path.join(directory, "loc/unexpected.json.gz"); - fs.writeFileSync(unexpected, "unexpected"); - assert.match((await generate()).stdout, /Generated diagnostics\./); - assert.equal(fs.existsSync(unexpected), false); - fs.rmSync(path.join(directory, "loc"), { recursive: true }); - assert.match((await generate()).stdout, /Generated diagnostics\./); - assert.deepEqual(fs.readFileSync(locales[0]), archive); - fs.rmSync(localized); - assert.match((await generate()).stdout, /Generated diagnostics\./); - assert.match((await generate("--force")).stdout, /Generated diagnostics\./); - assert.match((await generate()).stdout, /Diagnostics are up to date\./); - const forced = await x(process.execPath, [path.join(root, "tools/scripts/generateDiagnostics.mts"), "--outDir", directory], { - throwOnError: true, - nodeOptions: { cwd: root, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, - }); - assert.match(forced.stdout, /Generated diagnostics\./); - assert.match((await generate()).stdout, /Diagnostics are up to date\./); -}); - -test("Unicode generation skips unchanged tables and formatting", async () => { - const root = path.resolve(import.meta.dirname, "../.."); - const generate = () => x("go", ["-C", "./tsc", "generate", "./internal/stringutil"], { throwOnError: true, nodeOptions: { cwd: root } }); - await generate(); - const files = ["js_case_generated.go", "identifier_parts_generated.go"].map(file => path.join(root, "tsc/internal/stringutil", file)); - const timestamps = files.map(file => fs.statSync(file).mtimeMs); - const current = await generate(); - assert.match(current.stdout, /Unicode tables are up to date\./); - assert.doesNotMatch(current.stdout, /Formatted/); - assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); -}); - -test("Unicode generation repairs outputs and supports force", async context => { - const root = path.resolve(import.meta.dirname, "../.."); - const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_unicode-probe-")); - context.after(() => fs.rmSync(directory, { recursive: true, force: true })); - const command = [path.join(root, "tsc/internal/stringutil/_scripts/generate-unicode-data.mts"), "--outDir", directory]; - const generate = (...args: string[]) => x(process.execPath, [...command, ...args], { throwOnError: true, nodeOptions: { cwd: root } }); - await generate(); - const files = ["js_case_generated.go", "identifier_parts_generated.go"].map(file => path.join(directory, file)); - const originals = files.map(file => fs.readFileSync(file)); - const timestamps = files.map(file => fs.statSync(file).mtimeMs); - assert.match((await generate()).stdout, /Unicode tables are up to date\./); - assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); - for (const [index, file] of files.entries()) { - assert.deepEqual(originals[index], fs.readFileSync(path.join(root, "tsc/internal/stringutil", path.basename(file)))); - fs.writeFileSync(file, "modified"); - assert.match((await generate()).stdout, /Generated Unicode tables\./); - assert.deepEqual(fs.readFileSync(file), originals[index]); - fs.rmSync(file); - assert.match((await generate()).stdout, /Generated Unicode tables\./); - assert.deepEqual(fs.readFileSync(file), originals[index]); - } - const cacheFile = path.join(os.tmpdir(), "typescript-codegen", createHash("sha256").update(files[0]).digest("hex") + ".json"); - context.after(() => fs.rmSync(cacheFile, { force: true })); - fs.rmSync(cacheFile); - assert.match((await generate()).stdout, /Generated Unicode tables\./); - assert.match((await generate("--force")).stdout, /Generated Unicode tables\./); - assert.match((await generate()).stdout, /Unicode tables are up to date\./); - const forced = await x(process.execPath, command, { - throwOnError: true, - nodeOptions: { cwd: root, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, - }); - assert.match(forced.stdout, /Generated Unicode tables\./); - const overridden = await x(process.execPath, [...command, "--no-force"], { - throwOnError: true, - nodeOptions: { cwd: root, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, - }); - assert.match(overridden.stdout, /Unicode tables are up to date\./); -}); - -test("enum generation skips unchanged outputs and Go verification", async () => { - const root = path.resolve(import.meta.dirname, "../.."); - const generate = (force = false) => x("npx", ["hereby", "generate:enums", ...(force ? ["--force"] : [])], { throwOnError: true, nodeOptions: { cwd: root } }); - await generate(); - const files = [ - ...fs.globSync("packages/typescript/src/enums/*.ts", { cwd: root }), - "tsc/internal/api/enum_values_generated.go", - ].map(file => path.join(root, file)); - const timestamps = files.map(file => fs.statSync(file).mtimeMs); - const { stdout } = await generate(); - assert.match(stdout, /Enums are up to date\./); - assert.doesNotMatch(stdout, /Getting values from go/); - assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); - const forced = await generate(true); - assert.match(forced.stdout, /All generated values match Go\./); - assert.match((await generate()).stdout, /Enums are up to date\./); - const verifier = path.join(root, "tsc/internal/api/enum_values_generated.go"); - const cacheFile = path.join(os.tmpdir(), "typescript-codegen", createHash("sha256").update(verifier).digest("hex") + ".json"); - fs.rmSync(cacheFile); - const regenerated = await generate(); - assert.match(regenerated.stdout, /All generated values match Go\./); - assert.match((await generate()).stdout, /Enums are up to date\./); -}); - -test("AST generation forwards force to all schema generators", async () => { - const root = path.resolve(import.meta.dirname, "../.."); - const generate = (force = false) => x("npx", ["hereby", "generate:ast", ...(force ? ["--force"] : [])], { throwOnError: true, nodeOptions: { cwd: root } }); - await generate(); - const forced = await generate(true); - assert.match(forced.stdout, /Wrote .*encoder_generated\.go/); - assert.match(forced.stdout, /Wrote .*ast_generated\.go/); - assert.match(forced.stdout, /Generated .*ast\.generated\.ts/); - assert.doesNotMatch(forced.stdout, /\$ node .*tools\/scripts\/tsc\/generate\.ts/); - assert.doesNotMatch((await generate()).stdout, /(?:Wrote|Generated) /); -}); - -test("package generation forwards force to AST, encoder, and sync generators", async () => { - const root = path.resolve(import.meta.dirname, "../.."); - const generate = (force = false) => x("npm", ["run", "-w", "@typescript/typescript", "generate", ...(force ? ["--", "--force"] : [])], { throwOnError: true, nodeOptions: { cwd: root } }); - await generate(); - const forced = await generate(true); - assert.match(forced.stdout, /Generated .*ast\.generated\.ts/); - assert.match(forced.stdout, /Wrote .*encoder_generated\.go/); - assert.match(forced.stdout, /Formatting\.\.\./); - const current = await generate(); - assert.doesNotMatch(current.stdout, /(?:Wrote|Generated) |Formatting\.\.\./); - assert.match(current.stdout, /Sync API is up to date\./); -}); - -test("API protocol generation caches formatted output and supports force", async context => { - const root = path.resolve(import.meta.dirname, "../.."); - const output = path.join(root, "packages/typescript/src/api/proto.generated.ts"); - const generate = (force = false) => x("npx", ["hereby", "generate:api", ...(force ? ["--force"] : [])], { throwOnError: true, nodeOptions: { cwd: root } }); - await generate(); - const timestamp = fs.statSync(output).mtimeMs; - const current = await generate(); - assert.match(current.stdout, /API protocol is up to date\./); - assert.doesNotMatch(current.stdout, /\$ node .*generateAPI\.mts/); - assert.equal(fs.statSync(output).mtimeMs, timestamp); - const unrelated = path.join(root, "tsc/internal/parser/codegen_cache_probe.go"); - assert.equal(fs.existsSync(unrelated), false); - context.after(() => fs.rmSync(unrelated, { force: true })); - fs.writeFileSync(unrelated, "package parser\n"); - assert.match((await generate()).stdout, /API protocol is up to date\./); - fs.rmSync(unrelated); - const schema = path.join(root, "tsc/internal/api/requestfilesystem/codegen_cache_probe.go"); - assert.equal(fs.existsSync(schema), false); - const original = fs.readFileSync(output, "utf8"); - context.after(async () => { - if (fs.existsSync(schema)) { - fs.rmSync(schema); - await generate(true); - } - }); - fs.writeFileSync(schema, 'package requestfilesystem\n\nconst KindCodegenCacheProbe Kind = "codegen-cache-probe"\n'); - assert.match((await generate()).stdout, /Generated API protocol\./); - assert.match(fs.readFileSync(output, "utf8"), /"codegen-cache-probe"/); - fs.rmSync(schema); - assert.match((await generate()).stdout, /Generated API protocol\./); - assert.equal(fs.readFileSync(output, "utf8"), original); - assert.match((await generate(true)).stdout, /Generated API protocol\./); - assert.match((await generate()).stdout, /API protocol is up to date\./); - const direct = await x("node", ["./tools/scripts/generateAPI.mts"], { throwOnError: true, nodeOptions: { cwd: root } }); - assert.match(direct.stdout, /API protocol is up to date\./); - const nested = await x("go", ["-C", "./tsc", "generate", "./internal/api"], { - throwOnError: true, - nodeOptions: { cwd: root, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, - }); - assert.match(nested.stdout, /Generated API protocol\./); - assert.match((await generate()).stdout, /API protocol is up to date\./); -}); From 2607bb11ef8f39c638c6ce612bf6e4d296d1dd4f Mon Sep 17 00:00:00 2001 From: Wesley Wigham Date: Fri, 18 Sep 2026 19:49:22 -0700 Subject: [PATCH 3/6] Keep the commands we cache in the go:generate line --- Herebyfile.mjs | 24 ++- package.json | 1 + tools/scripts/gen/cache.mts | 120 +++++++++++ tools/scripts/gen/cache.test.mts | 158 ++++++++++++++ tools/scripts/gen/generateAPI.mts | 38 ---- tools/scripts/gen/generateBundled.mts | 40 ---- tools/scripts/gen/generateDiagnostics.mts | 55 ----- tools/scripts/gen/generateMoq.mts | 47 ----- tools/scripts/gen/generateStringer.mts | 49 ----- tools/scripts/gen/generatedFile.test.mts | 197 +++++++++++------- tools/scripts/gen/utils.mts | 23 +- tools/scripts/tsc/generate-go-ast.ts | 2 +- tsc/internal/ast/kind_generated.go | 2 +- tsc/internal/bundled/bundled.go | 2 +- tsc/internal/checker/types.go | 2 +- tsc/internal/core/compileroptions.go | 4 +- tsc/internal/core/languagevariant.go | 2 +- tsc/internal/core/scriptkind.go | 2 +- tsc/internal/core/tristate.go | 2 +- tsc/internal/diagnostics/diagnostics.go | 4 +- tsc/internal/ls/autoimport/export.go | 2 +- tsc/internal/project/project.go | 2 +- .../projecttestutil/projecttestutil.go | 4 +- tsc/internal/vfs/vfs.go | 2 +- tsc/internal/vfs/vfsmatch/vfsmatch.go | 2 +- 25 files changed, 442 insertions(+), 344 deletions(-) create mode 100644 tools/scripts/gen/cache.mts create mode 100644 tools/scripts/gen/cache.test.mts delete mode 100644 tools/scripts/gen/generateAPI.mts delete mode 100644 tools/scripts/gen/generateBundled.mts delete mode 100644 tools/scripts/gen/generateDiagnostics.mts delete mode 100644 tools/scripts/gen/generateMoq.mts delete mode 100644 tools/scripts/gen/generateStringer.mts diff --git a/Herebyfile.mjs b/Herebyfile.mjs index 391605d9b8154..82355c02b0491 100644 --- a/Herebyfile.mjs +++ b/Herebyfile.mjs @@ -998,8 +998,28 @@ export const generateAST = task({ }); async function runGenerateAPI() { - const { default: generate } = await import("./tools/scripts/gen/generateAPI.mts"); - await generate(!!options.force); + const { default: cache } = await import("./tools/scripts/gen/cache.mts"); + await cache({ + cwd: __dirname, + inputs: [ + __filename, + "tsc/internal/api/*.go", + "tsc/internal/api/requestfilesystem/*.go", + "tsc/internal/core/*.go", + "tsc/internal/checker/types.go", + "tsc/internal/diagnostics/diagnostics.go", + "tsc/internal/tspath/path.go", + "tools/gen-proto/*.go", + ], + exclude: ["**/*_test.go", "**/*_generated.go"], + envInputs: [], + outputs: ["packages/typescript/src/api/proto.generated.ts"], + commands: [ + ["go", "-C", "./tools", "run", "./gen-proto", "../tsc/internal/api/proto.go", "../packages/typescript/src/api/proto.generated.ts"], + ["dprint", "fmt", "packages/typescript/src/api/proto.generated.ts"], + ], + force: !!options.force, + }); } export const generateAPI = task({ diff --git a/package.json b/package.json index fbdcc207eed23..fc7edb42f8bb8 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "format": "hereby format", "check:format": "hereby check:format", "generate": "hereby generate", + "cache": "node tools/scripts/gen/cache.mts", "tidy": "hereby tidy", "extension:build": "npm run -w native-preview build", "extension:watch": "npm run -w native-preview watch", diff --git a/tools/scripts/gen/cache.mts b/tools/scripts/gen/cache.mts new file mode 100644 index 0000000000000..44241d8d248f9 --- /dev/null +++ b/tools/scripts/gen/cache.mts @@ -0,0 +1,120 @@ +import { createHash } from "node:crypto"; +import * as fs from "node:fs"; +import path from "node:path"; +import { x } from "tinyexec"; +import { GeneratedFile } from "./generatedFile.mts"; +import { + goInputs, + parseGeneratorArgs, + repoRoot, +} from "./utils.mts"; + +export interface CacheOptions { + inputs: string[]; + outputs: string[]; + commands: string[][]; + cwd?: string; + exclude?: string[]; + envInputs?: string[]; + force?: boolean; +} + +function expand(patterns: string[], cwd: string, exclude: string[] = []) { + return [...new Set(fs.globSync(patterns, { cwd, exclude }).map(file => path.resolve(cwd, file)))].sort(); +} + +export default async function cache({ + inputs, + outputs, + commands, + cwd = process.cwd(), + exclude = [], + envInputs = ["GOOS", "GOARCH", "GOFLAGS", "GOTOOLCHAIN", "GOEXPERIMENT", "CGO_ENABLED", "GOWORK"], + force = false, +}: CacheOptions): Promise { + if (!outputs.length || !commands.length || commands.some(command => !command.length)) { + throw new Error("Cached generation requires outputs and nonempty commands."); + } + cwd = path.resolve(cwd); + const snapshotInputs = () => { + const entries = [ + ...new Set([ + import.meta.filename, + path.join(import.meta.dirname, "generatedFile.mts"), + path.join(repoRoot, "package-lock.json"), + path.join(repoRoot, ".dprint.jsonc"), + ...goInputs(), + ...inputs.flatMap(pattern => { + const matches = expand([pattern], cwd, exclude); + if (!matches.length) throw new Error(`No inputs matched: ${pattern}`); + return matches; + }), + ]), + ].sort(); + const files: string[] = []; + const hash = createHash("sha256"); + for (const file of entries) { + if (fs.statSync(file).isDirectory()) { + hash.update(JSON.stringify([file, "directory"])); + } + else { + files.push(file); + hash.update(JSON.stringify([file, createHash("sha256").update(fs.readFileSync(file)).digest("hex")])); + } + } + return { files, hash: hash.digest("hex") }; + }; + const before = snapshotInputs(); + const outputFiles = () => expand(outputs, cwd).filter(file => fs.statSync(file).isFile()); + const artifacts = (files: string[]) => + files.map(file => + new GeneratedFile(file, before.files, undefined, { + cwd, + commands, + inputs, + outputs, + exclude, + inputHash: before.hash, + outputFiles: files, + environment: envInputs.map(name => [name, process.env[name]]), + }) + ); + const previous = artifacts(outputFiles()); + const complete = () => outputs.every(pattern => expand([pattern], cwd).some(file => fs.statSync(file).isFile())); + if (complete() && previous.every(file => file.isCurrent(force))) { + console.log("Codegen outputs are up to date."); + return true; + } + + for (const file of previous) file.invalidate(); + for (const [command, ...args] of commands) { + await x(command, args, { throwOnError: true, nodeOptions: { cwd, stdio: "inherit" } }); + } + if (!complete()) throw new Error(`Generation did not produce all declared outputs: ${outputs.join(", ")}`); + if (snapshotInputs().hash === before.hash) { + for (const file of artifacts(outputFiles())) file.markCurrent(); + } + console.log("Generated codegen outputs."); + return false; +} + +if (process.argv[1] === import.meta.filename) { + const args = process.argv.slice(2); + const cwd = process.env.npm_lifecycle_event === "cache" ? process.env.INIT_CWD ?? process.cwd() : process.cwd(); + const commandStart = args.indexOf("--command"); + const { values, force } = parseGeneratorArgs({ + input: { type: "string", multiple: true }, + output: { type: "string", multiple: true }, + exclude: { type: "string", multiple: true }, + env: { type: "string", multiple: true }, + cwd: { type: "string" }, + }, commandStart < 0 ? args : args.slice(0, commandStart)); + const commands: string[][] = []; + if (commandStart >= 0) { + for (const arg of args.slice(commandStart)) { + if (arg === "--command") commands.push([]); + else commands[commands.length - 1].push(arg); + } + } + await cache({ inputs: values.input ?? [], outputs: values.output ?? [], commands, cwd: path.resolve(cwd, values.cwd ?? "."), exclude: values.exclude, envInputs: values.env, force }); +} diff --git a/tools/scripts/gen/cache.test.mts b/tools/scripts/gen/cache.test.mts new file mode 100644 index 0000000000000..83e7426f309ff --- /dev/null +++ b/tools/scripts/gen/cache.test.mts @@ -0,0 +1,158 @@ +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { x } from "tinyexec"; +import cache from "./cache.mts"; + +test("npm cache preserves the caller directory and command arguments without task logging", async context => { + const root = path.resolve(import.meta.dirname, "../../.."); + const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_cache probe-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true, maxRetries: 10 })); + fs.writeFileSync(path.join(directory, "input.txt"), "input"); + fs.writeFileSync(path.join(directory, "generate.cjs"), "const fs = require('node:fs'); fs.writeFileSync('output.txt', fs.readFileSync('input.txt') + JSON.stringify(process.argv.slice(2)));\n"); + fs.writeFileSync(path.join(directory, "finish.cjs"), "require('node:fs').appendFileSync('output.txt', ' finished');\n"); + const generate = (flags: string[] = [], env = process.env, failure = false) => + x("npm", [ + "run", + "--silent", + "cache", + "--", + "--input", + "input.txt", + "--input", + "*.cjs", + "--output", + "output.txt", + ...flags, + "--command", + process.execPath, + "generate.cjs", + "--help", + "--force", + "space value", + "--command", + process.execPath, + failure ? "missing.cjs" : "finish.cjs", + ], { throwOnError: true, nodeOptions: { cwd: directory, env: { ...env, INIT_CWD: root } } }); + const output = path.join(directory, "output.txt"); + assert.equal((await generate()).stdout.trim(), "Generated codegen outputs."); + assert.equal(fs.readFileSync(output, "utf8"), 'input["--help","--force","space value"] finished'); + const timestamp = fs.statSync(output).mtimeMs; + assert.equal((await generate()).stdout.trim(), "Codegen outputs are up to date."); + assert.equal(fs.statSync(output).mtimeMs, timestamp); + assert.doesNotMatch((await generate(["--force"])).stdout, /up to date/); + const forced = { ...process.env, TSGO_HEREBY_FORCE: "1" }; + assert.match((await generate(["--no-force"], forced)).stdout, /up to date/); + assert.doesNotMatch((await generate([], forced)).stdout, /up to date/); + await assert.rejects(async () => await generate([], process.env, true)); + assert.doesNotMatch((await generate()).stdout, /up to date/); + assert.match((await generate()).stdout, /up to date/); +}); + +test("cache commands track inputs, outputs, arguments, and failures", async context => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-cache-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true, maxRetries: 10 })); + const input = path.join(directory, "input.txt"); + const output = path.join(directory, "output.txt"); + fs.writeFileSync(input, "first"); + fs.writeFileSync(path.join(directory, "generate.cjs"), "const fs = require('node:fs'); fs.writeFileSync('output.txt', fs.readFileSync('input.txt'));\n"); + fs.writeFileSync(path.join(directory, "finish.cjs"), "const fs = require('node:fs'); fs.appendFileSync('output.txt', process.argv[2]); fs.writeFileSync('extra.txt', 'extra');\n"); + const generate = (suffix = " done", flags: string[] = [], failure = false, env = process.env) => + x(process.execPath, [ + path.join(import.meta.dirname, "cache.mts"), + "--env", + "TSGO_CACHE_PROBE", + "--input", + "input*.txt", + "--input", + "*.cjs", + "--output", + "output.txt", + "--output", + "extra*.txt", + ...flags, + "--command", + process.execPath, + "generate.cjs", + "--command", + process.execPath, + failure ? "missing.cjs" : "finish.cjs", + suffix, + ], { throwOnError: true, nodeOptions: { cwd: directory, env } }); + await generate(); + assert.equal(fs.readFileSync(output, "utf8"), "first done"); + const timestamp = fs.statSync(output).mtimeMs; + assert.match((await generate()).stdout, /up to date/); + assert.equal(fs.statSync(output).mtimeMs, timestamp); + fs.writeFileSync(input, "second"); + await generate(); + assert.equal(fs.readFileSync(output, "utf8"), "second done"); + const addedInput = path.join(directory, "input-added.txt"); + fs.writeFileSync(addedInput, "added"); + assert.doesNotMatch((await generate()).stdout, /up to date/); + fs.rmSync(addedInput); + assert.doesNotMatch((await generate()).stdout, /up to date/); + fs.rmSync(path.join(directory, "extra.txt")); + assert.doesNotMatch((await generate()).stdout, /up to date/); + fs.writeFileSync(path.join(directory, "extra-added.txt"), "extra"); + assert.doesNotMatch((await generate()).stdout, /up to date/); + fs.writeFileSync(output, "edited"); + assert.doesNotMatch((await generate()).stdout, /up to date/); + assert.doesNotMatch((await generate(" changed")).stdout, /up to date/); + assert.equal(fs.readFileSync(output, "utf8"), "second changed"); + assert.match((await generate(" changed")).stdout, /up to date/); + assert.doesNotMatch((await generate(" changed", ["--force"])).stdout, /up to date/); + await assert.rejects(async () => await generate(" changed", ["--force"], true)); + assert.doesNotMatch((await generate(" changed")).stdout, /up to date/); + assert.match((await generate(" changed")).stdout, /up to date/); + const forced = { ...process.env, TSGO_HEREBY_FORCE: "1" }; + assert.match((await generate(" changed", ["--no-force"], false, forced)).stdout, /up to date/); + assert.doesNotMatch((await generate(" changed", [], false, forced)).stdout, /up to date/); + assert.match((await generate(" changed")).stdout, /up to date/); + const environment = { ...process.env, TSGO_CACHE_PROBE: "changed" }; + assert.doesNotMatch((await generate(" changed", [], false, environment)).stdout, /up to date/); + assert.match((await generate(" changed", [], false, environment)).stdout, /up to date/); + assert.doesNotMatch((await generate("--force")).stdout, /up to date/); + assert.equal(fs.readFileSync(output, "utf8"), "second--force"); + assert.match((await generate("--force")).stdout, /up to date/); +}); + +test("cache commands reject missing inputs, outputs, and empty commands", async context => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-cache-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const options = { + cwd: directory, + inputs: ["input.txt"], + outputs: ["output.txt"], + commands: [[process.execPath, "-e", "require('node:fs').writeFileSync('output.txt', 'generated')"]], + }; + await assert.rejects(cache(options), /No inputs matched/); + assert.equal(fs.existsSync(path.join(directory, "output.txt")), false); + fs.writeFileSync(path.join(directory, "input.txt"), "input"); + await assert.rejects(cache({ ...options, inputs: ["missing*.txt"] }), /No inputs matched/); + await assert.rejects(cache({ ...options, commands: [] }), /nonempty commands/); + await assert.rejects(cache({ ...options, commands: [[]] }), /nonempty commands/); + await assert.rejects(cache({ ...options, outputs: [] }), /requires outputs/); + await assert.rejects(cache({ ...options, outputs: ["output.txt", "missing.txt"] }), /did not produce all declared outputs/); + assert.equal(await cache(options), false); + assert.equal(await cache(options), true); +}); + +test("cache commands do not cache inputs changed during execution", async context => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-cache-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + for (const change of ["fs.writeFileSync('input.txt', 'changed')", "fs.writeFileSync('input-added.txt', 'added')"]) { + fs.writeFileSync(path.join(directory, "input.txt"), "initial"); + const options = { + cwd: directory, + inputs: ["input*.txt"], + outputs: ["output.txt"], + commands: [[process.execPath, "-e", `const fs = require('node:fs'); fs.writeFileSync('output.txt', 'generated'); ${change};`]], + }; + assert.equal(await cache(options), false); + assert.equal(await cache(options), false); + assert.equal(await cache(options), true); + } +}); diff --git a/tools/scripts/gen/generateAPI.mts b/tools/scripts/gen/generateAPI.mts deleted file mode 100644 index 43a08bb8791ed..0000000000000 --- a/tools/scripts/gen/generateAPI.mts +++ /dev/null @@ -1,38 +0,0 @@ -import path from "node:path"; -import { GeneratedFile } from "./generatedFile.mts"; -import { - formatFiles, - globInputs, - goInputs, - parseGeneratorArgs, - repoRoot, - runGo, -} from "./utils.mts"; - -export default async function generateAPI(force = false) { - const inputs = globInputs([ - "tsc/internal/api/*.go", - "tsc/internal/api/requestfilesystem/*.go", - "tsc/internal/core/*.go", - "tsc/internal/checker/types.go", - "tsc/internal/diagnostics/diagnostics.go", - "tsc/internal/tspath/path.go", - "tools/gen-proto/*.go", - ], ["**/*_test.go", "**/*_generated.go"]); - const generated = new GeneratedFile(path.join(repoRoot, "packages/typescript/src/api/proto.generated.ts"), [import.meta.filename, ...goInputs(), ...inputs]); - if (generated.isCurrent(force)) { - console.log("API protocol is up to date."); - return; - } - - generated.invalidate(); - console.log("Generating API protocol..."); - await runGo(["-C", "./tools", "run", "./gen-proto", "../tsc/internal/api/proto.go", generated.fileName]); - await formatFiles([generated.fileName]); - generated.markCurrent(); - console.log("Generated API protocol."); -} - -if (process.argv[1] === import.meta.filename) { - await generateAPI(parseGeneratorArgs({}).force); -} diff --git a/tools/scripts/gen/generateBundled.mts b/tools/scripts/gen/generateBundled.mts deleted file mode 100644 index d07f5a53a1038..0000000000000 --- a/tools/scripts/gen/generateBundled.mts +++ /dev/null @@ -1,40 +0,0 @@ -import * as fs from "node:fs"; -import path from "node:path"; -import { GeneratedFile } from "./generatedFile.mts"; -import { - goEnvironment, - goInputs, - parseGeneratorArgs, - repoRoot, - runGo, -} from "./utils.mts"; - -const sourceDirectory = path.join(repoRoot, "tsc/internal/bundled"); - -export default async function generateBundled(force = false, directory = sourceDirectory) { - directory = path.resolve(directory); - const entries = fs.readdirSync(path.join(directory, "libs"), { withFileTypes: true }).sort((left, right) => left.name < right.name ? -1 : left.name > right.name ? 1 : 0); - const inputs = [ - import.meta.filename, - path.join(sourceDirectory, "generate.go"), - path.join(directory, "CopyrightNotice.txt"), - ...entries.filter(entry => !entry.isDirectory()).map(entry => path.join(directory, "libs", entry.name)), - ...goInputs(), - ]; - const key = [goEnvironment(), entries.map(entry => [entry.name, entry.isDirectory()])]; - const generatedFiles = ["libs_generated.go", "embed_generated.go"].map(file => new GeneratedFile(path.join(directory, file), inputs, undefined, key)); - if (generatedFiles.every(file => file.isCurrent(force))) { - console.log("Bundled libraries are up to date."); - return; - } - - for (const file of generatedFiles) file.invalidate(); - await runGo(["run", path.join(sourceDirectory, "generate.go")], directory); - for (const file of generatedFiles) file.markCurrent(); - console.log("Generated bundled libraries."); -} - -if (process.argv[1] === import.meta.filename) { - const { values, force } = parseGeneratorArgs({ directory: { type: "string" } }); - await generateBundled(force, values.directory); -} diff --git a/tools/scripts/gen/generateDiagnostics.mts b/tools/scripts/gen/generateDiagnostics.mts deleted file mode 100644 index 61f16d86c924e..0000000000000 --- a/tools/scripts/gen/generateDiagnostics.mts +++ /dev/null @@ -1,55 +0,0 @@ -import * as fs from "node:fs"; -import path from "node:path"; -import { GeneratedFile } from "./generatedFile.mts"; -import { - formatFiles, - globInputs, - goEnvironment, - goInputs, - parseGeneratorArgs, - repoRoot, - runGo, -} from "./utils.mts"; - -const sourceDirectory = path.join(repoRoot, "tsc/internal/diagnostics"); - -export default async function generateDiagnostics(force = false, outDir = sourceDirectory) { - outDir = path.resolve(outDir); - const inputs = [ - import.meta.filename, - path.join(sourceDirectory, "generate.go"), - path.join(sourceDirectory, "diagnosticMessages.json"), - path.join(sourceDirectory, "extraDiagnosticMessages.json"), - ...goInputs(), - ...globInputs([ - "tsc/internal/{collections,json}/*.go", - "tsc/internal/locale/lcl/*/diagnosticMessages/diagnosticMessages.generated.json.lcl", - ], ["**/*_test.go"]), - ]; - const environment = goEnvironment(); - const generated = new GeneratedFile(path.join(outDir, "diagnostics_generated.go"), inputs, undefined, environment); - const localized = new GeneratedFile(path.join(outDir, "loc_generated.go"), inputs, undefined, environment); - const goFiles = [generated, localized]; - const localeFiles = () => - [...fs.readFileSync(localized.fileName, "utf8").matchAll(/^\/\/go:embed (loc\/[^\r\n]+\.json\.gz)\r?$/gm)] - .map(match => new GeneratedFile(path.join(outDir, match[1]), inputs, undefined, environment)); - if (goFiles.every(file => file.isCurrent(force))) { - const locales = localeFiles(); - if (locales.length === fs.globSync("loc/*.json.gz", { cwd: outDir }).length && locales.every(file => file.isCurrent())) { - console.log("Diagnostics are up to date."); - return; - } - } - - for (const file of goFiles) file.invalidate(); - fs.mkdirSync(outDir, { recursive: true }); - await runGo(["run", "generate.go", "-diagnostics", generated.fileName, "-loc", localized.fileName, "-locdir", path.join(outDir, "loc")], sourceDirectory); - await formatFiles(goFiles.map(file => file.fileName)); - for (const file of [...localeFiles(), ...goFiles]) file.markCurrent(); - console.log("Generated diagnostics."); -} - -if (process.argv[1] === import.meta.filename) { - const { values, force } = parseGeneratorArgs({ outDir: { type: "string" } }); - await generateDiagnostics(force, values.outDir); -} diff --git a/tools/scripts/gen/generateMoq.mts b/tools/scripts/gen/generateMoq.mts deleted file mode 100644 index 16f370fc18c08..0000000000000 --- a/tools/scripts/gen/generateMoq.mts +++ /dev/null @@ -1,47 +0,0 @@ -import path from "node:path"; -import { GeneratedFile } from "./generatedFile.mts"; -import { - formatFiles, - goEnvironment, - goInputs, - parseGeneratorArgs, - runGo, -} from "./utils.mts"; - -export default async function generateMoq(args: string[], cwd = process.cwd()) { - const { values, positionals, toolArgs: moqArgs, force } = parseGeneratorArgs( - { - out: { type: "string" }, - pkg: { type: "string" }, - fmt: { type: "string" }, - stub: { type: "boolean" }, - input: { type: "string", multiple: true }, - }, - args, - true, - ); - if (!values.out || !values.input?.length || positionals.length < 2) { - throw new Error("Usage: generateMoq.mts --input=source.go -out=mock.go [moq options] [--force] "); - } - const inputs = [ - import.meta.filename, - ...goInputs(), - ...values.input.map(file => path.resolve(cwd, file)), - ...(process.env.GOFILE ? [path.resolve(cwd, process.env.GOFILE)] : []), - ]; - const generated = new GeneratedFile(path.resolve(cwd, values.out), inputs, undefined, [moqArgs, goEnvironment()]); - if (generated.isCurrent(force)) { - console.log("Moq output is up to date."); - return; - } - - generated.invalidate(); - await runGo(["tool", "github.com/matryer/moq", ...moqArgs], cwd); - await formatFiles([generated.fileName]); - generated.markCurrent(); - console.log("Generated moq output."); -} - -if (process.argv[1] === import.meta.filename) { - await generateMoq(process.argv.slice(2)); -} diff --git a/tools/scripts/gen/generateStringer.mts b/tools/scripts/gen/generateStringer.mts deleted file mode 100644 index cfe67017cf9e3..0000000000000 --- a/tools/scripts/gen/generateStringer.mts +++ /dev/null @@ -1,49 +0,0 @@ -import path from "node:path"; -import { GeneratedFile } from "./generatedFile.mts"; -import { - formatFiles, - goEnvironment, - goInputs, - parseGeneratorArgs, - runGo, -} from "./utils.mts"; - -export default async function generateStringer(args: string[], cwd = process.cwd()) { - const { values, positionals, toolArgs: stringerArgs, force } = parseGeneratorArgs( - { - type: { type: "string" }, - output: { type: "string" }, - trimprefix: { type: "string" }, - linecomment: { type: "boolean" }, - tags: { type: "string" }, - input: { type: "string" }, - }, - args, - true, - ); - const input = values.input ?? process.env.GOFILE; - if (!values.type || !values.output || !input || positionals.length) { - throw new Error("Usage: generateStringer.mts -type=Type -output=file.go [--input=source.go] [--force] (input defaults to GOFILE)"); - } - const output = path.resolve(cwd, values.output); - const inputs = [ - import.meta.filename, - ...goInputs(), - path.resolve(cwd, input), - ]; - const generated = new GeneratedFile(output, inputs, undefined, [stringerArgs, goEnvironment()]); - if (generated.isCurrent(force)) { - console.log("Stringer output is up to date."); - return; - } - - generated.invalidate(); - await runGo(["tool", "golang.org/x/tools/cmd/stringer", ...stringerArgs], cwd); - await formatFiles([output]); - generated.markCurrent(); - console.log("Generated stringer output."); -} - -if (process.argv[1] === import.meta.filename) { - await generateStringer(process.argv.slice(2)); -} diff --git a/tools/scripts/gen/generatedFile.test.mts b/tools/scripts/gen/generatedFile.test.mts index a53c85a5eb282..881111faf8c03 100644 --- a/tools/scripts/gen/generatedFile.test.mts +++ b/tools/scripts/gen/generatedFile.test.mts @@ -5,17 +5,29 @@ import os from "node:os"; import path from "node:path"; import { test } from "node:test"; import { x } from "tinyexec"; +import type { CacheOptions } from "./cache.mts"; import { GeneratedFile } from "./generatedFile.mts"; -test("generator options preserve tool arguments and force overrides", async () => { +function generatorArgs(options: CacheOptions, flags: string[] = []): string[] { + return [ + path.join(import.meta.dirname, "cache.mts"), + ...(options.cwd ? ["--cwd", options.cwd] : []), + ...options.inputs.flatMap(file => ["--input", file]), + ...options.outputs.flatMap(file => ["--output", file]), + ...(options.exclude ?? []).flatMap(pattern => ["--exclude", pattern]), + ...(options.envInputs ?? []).flatMap(name => ["--env", name]), + ...flags, + ...options.commands.flatMap(command => ["--command", ...command]), + ]; +} + +test("generator options preserve repeated inputs and force overrides", async () => { const { parseGeneratorArgs, resolveForce } = await import("./utils.mts"); const options = { type: { type: "string" }, input: { type: "string", multiple: true } } as const; - const args = ["-type", "Kind", "--input=kind.go", "--input", "extra.go", "--force", ".", "Kind:Mock"]; - const parsed = parseGeneratorArgs(options, args, true); + const args = ["--type", "Kind", "--input=kind.go", "--input", "extra.go", "--force"]; + const parsed = parseGeneratorArgs(options, args); assert.equal(parsed.values.type, "Kind"); assert.deepEqual(parsed.values.input, ["kind.go", "extra.go"]); - assert.deepEqual(parsed.positionals, [".", "Kind:Mock"]); - assert.deepEqual(parsed.toolArgs, ["-type", "Kind", ".", "Kind:Mock"]); assert.equal(parsed.force, true); assert.equal(resolveForce(undefined, "ON"), true); assert.equal(resolveForce(undefined, "0"), false); @@ -141,7 +153,7 @@ test("bundled generation skips unchanged library outputs", async () => { await generate(); const files = ["libs_generated.go", "embed_generated.go"].map(file => path.join(root, "tsc/internal/bundled", file)); const timestamps = files.map(file => fs.statSync(file).mtimeMs); - assert.match((await generate()).stdout, /Bundled libraries are up to date\./); + assert.match((await generate()).stdout, /Codegen outputs are up to date\./); assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); }); @@ -156,61 +168,67 @@ test("bundled generation tracks libraries, validates inputs, and supports force" const library = path.join(libs, "lib.first.d.ts"); const content = "Fixture notice\n\n\ndeclare const first: string;\n"; fs.writeFileSync(library, content); - const command = [path.join(root, "tools/scripts/gen/generateBundled.mts"), "--directory", directory]; - const generate = (...args: string[]) => x(process.execPath, [...command, ...args], { throwOnError: true, nodeOptions: { cwd: root } }); + const options = { + cwd: directory, + inputs: [path.join(root, "tsc/internal/bundled/generate.go"), "CopyrightNotice.txt", "libs/*"], + outputs: ["libs_generated.go", "embed_generated.go"], + commands: [["go", "run", path.join(root, "tsc/internal/bundled/generate.go")]], + }; + const command = generatorArgs(options); + const generate = (...args: string[]) => x(process.execPath, generatorArgs(options, args), { throwOnError: true, nodeOptions: { cwd: root } }); await generate(); const files = ["libs_generated.go", "embed_generated.go"].map(file => path.join(directory, file)); const originals = files.map(file => fs.readFileSync(file)); - assert.match((await generate()).stdout, /Bundled libraries are up to date\./); + assert.match((await generate()).stdout, /Codegen outputs are up to date\./); const added = path.join(libs, "lib.second.d.ts"); fs.writeFileSync(added, content); - assert.match((await generate()).stdout, /Generated bundled libraries\./); + assert.match((await generate()).stdout, /Generated codegen outputs\./); for (const file of files) assert.match(fs.readFileSync(file, "utf8"), /lib\.second\.d\.ts/); fs.rmSync(added); - assert.match((await generate()).stdout, /Generated bundled libraries\./); + assert.match((await generate()).stdout, /Generated codegen outputs\./); for (const [index, file] of files.entries()) { assert.deepEqual(fs.readFileSync(file), originals[index]); fs.writeFileSync(file, "modified"); - assert.match((await generate()).stdout, /Generated bundled libraries\./); + assert.match((await generate()).stdout, /Generated codegen outputs\./); assert.deepEqual(fs.readFileSync(file), originals[index]); fs.rmSync(file, { recursive: true, maxRetries: 10 }); - assert.match((await generate()).stdout, /Generated bundled libraries\./); + assert.match((await generate()).stdout, /Generated codegen outputs\./); assert.deepEqual(fs.readFileSync(file), originals[index]); } fs.writeFileSync(library, content.replace("string", "number")); - assert.match((await generate()).stdout, /Generated bundled libraries\./); + assert.match((await generate()).stdout, /Generated codegen outputs\./); fs.writeFileSync(library, content.replaceAll("\n", "\r\n")); await assert.rejects(async () => await generate(), error => { assert.match((error as { output: { stderr: string; }; }).output.stderr, /must use LF line endings/); return true; }); fs.writeFileSync(library, content); - assert.match((await generate()).stdout, /Generated bundled libraries\./); + assert.match((await generate()).stdout, /Generated codegen outputs\./); fs.mkdirSync(path.join(libs, "unexpected")); await assert.rejects(async () => await generate(), error => { assert.match((error as { output: { stderr: string; }; }).output.stderr, /unexpected entry/); return true; }); fs.rmdirSync(path.join(libs, "unexpected")); - assert.match((await generate()).stdout, /Generated bundled libraries\./); + assert.match((await generate()).stdout, /Generated codegen outputs\./); fs.writeFileSync(notice, "Changed fixture notice\n\n"); await assert.rejects(async () => await generate(), error => { assert.match((error as { output: { stderr: string; }; }).output.stderr, /must start with/); return true; }); fs.writeFileSync(notice, "Fixture notice\n\n"); - assert.match((await generate()).stdout, /Generated bundled libraries\./); - assert.match((await generate("--force")).stdout, /Generated bundled libraries\./); - assert.match((await generate()).stdout, /Bundled libraries are up to date\./); + assert.match((await generate()).stdout, /Generated codegen outputs\./); + assert.match((await generate("--force")).stdout, /Generated codegen outputs\./); + assert.match((await generate()).stdout, /Codegen outputs are up to date\./); const forced = await x(process.execPath, command, { throwOnError: true, nodeOptions: { cwd: root, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, }); - assert.match(forced.stdout, /Generated bundled libraries\./); - assert.match((await generate()).stdout, /Bundled libraries are up to date\./); + assert.match(forced.stdout, /Generated codegen outputs\./); + assert.match((await generate()).stdout, /Codegen outputs are up to date\./); }); -test("stringer generation tracks GOFILE, options, output, and force", async context => { +test("stringer generation tracks declared inputs, options, output, and force", async context => { const root = path.resolve(import.meta.dirname, "../../.."); const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_stringer-probe-")); context.after(() => fs.rmSync(directory, { recursive: true, force: true })); @@ -218,51 +236,60 @@ test("stringer generation tracks GOFILE, options, output, and force", async cont const extra = path.join(directory, "extra.go"); const output = path.join(directory, "kind_stringer_generated.go"); fs.writeFileSync(source, "package probe\n\ntype Kind int\n\nconst KindFirst Kind = 0\n"); + const command = (...args: string[]) => + generatorArgs({ + cwd: directory, + inputs: ["kind.go"], + outputs: ["kind_stringer_generated.go"], + commands: [ + ["go", "tool", "golang.org/x/tools/cmd/stringer", "-type=Kind", "-output=kind_stringer_generated.go", ...args.filter(arg => arg !== "--force")], + ["dprint", "fmt", "kind_stringer_generated.go"], + ], + }, args.filter(arg => arg === "--force")); const generate = (...args: string[]) => - x(process.execPath, [path.join(root, "tools/scripts/gen/generateStringer.mts"), "-type=Kind", "-output=kind_stringer_generated.go", ...args], { + x(process.execPath, command(...args), { throwOnError: true, nodeOptions: { cwd: directory, env: { ...process.env, GOFILE: "kind.go" } }, }); await generate(); const timestamp = fs.statSync(output).mtimeMs; - assert.match((await generate()).stdout, /Stringer output is up to date\./); + assert.match((await generate()).stdout, /Codegen outputs are up to date\./); assert.equal(fs.statSync(output).mtimeMs, timestamp); fs.writeFileSync(extra, "package probe\n\nconst Unrelated = 1\n"); - assert.match((await generate()).stdout, /Stringer output is up to date\./); + assert.match((await generate()).stdout, /Codegen outputs are up to date\./); fs.appendFileSync(source, "\nconst KindSecond Kind = 1\n"); - assert.match((await generate()).stdout, /Generated stringer output\./); + assert.match((await generate()).stdout, /Generated codegen outputs\./); assert.match(fs.readFileSync(output, "utf8"), /KindSecond/); fs.rmSync(extra); - assert.match((await generate()).stdout, /Stringer output is up to date\./); + assert.match((await generate()).stdout, /Codegen outputs are up to date\./); fs.writeFileSync(source, "package probe\n\ntype Kind int\n\nconst KindFirst Kind = 0\n"); - assert.match((await generate()).stdout, /Generated stringer output\./); + assert.match((await generate()).stdout, /Generated codegen outputs\./); assert.doesNotMatch(fs.readFileSync(output, "utf8"), /KindSecond/); - assert.match((await generate("-trimprefix=Kind")).stdout, /Generated stringer output\./); + assert.match((await generate("-trimprefix=Kind")).stdout, /Generated codegen outputs\./); assert.match(fs.readFileSync(output, "utf8"), /_Kind_name = "First"/); - assert.match((await generate("-trimprefix=Kind")).stdout, /Stringer output is up to date\./); + assert.match((await generate("-trimprefix=Kind")).stdout, /Codegen outputs are up to date\./); fs.appendFileSync(output, "\n"); - assert.match((await generate("-trimprefix=Kind")).stdout, /Generated stringer output\./); + assert.match((await generate("-trimprefix=Kind")).stdout, /Generated codegen outputs\./); fs.rmSync(output); - assert.match((await generate("-trimprefix=Kind")).stdout, /Generated stringer output\./); - assert.match((await generate("-trimprefix=Kind", "--force")).stdout, /Generated stringer output\./); - assert.match((await generate("-trimprefix=Kind")).stdout, /Stringer output is up to date\./); - const command = [path.join(root, "tools/scripts/gen/generateStringer.mts"), "-type=Kind", "-output=kind_stringer_generated.go", "-trimprefix=Kind"]; - const explicit = await x(process.execPath, [...command, "--input", "kind.go"], { + assert.match((await generate("-trimprefix=Kind")).stdout, /Generated codegen outputs\./); + assert.match((await generate("-trimprefix=Kind", "--force")).stdout, /Generated codegen outputs\./); + assert.match((await generate("-trimprefix=Kind")).stdout, /Codegen outputs are up to date\./); + const explicit = await x(process.execPath, command("-trimprefix=Kind"), { throwOnError: true, nodeOptions: { cwd: directory, env: { ...process.env, GOFILE: "" } }, }); - assert.match(explicit.stdout, /Stringer output is up to date\./); - const forced = await x(process.execPath, command, { + assert.match(explicit.stdout, /Codegen outputs are up to date\./); + const forced = await x(process.execPath, command("-trimprefix=Kind"), { throwOnError: true, nodeOptions: { cwd: directory, env: { ...process.env, GOFILE: "kind.go", TSGO_HEREBY_FORCE: "1" } }, }); - assert.match(forced.stdout, /Generated stringer output\./); - assert.match((await generate("-trimprefix=Kind")).stdout, /Stringer output is up to date\./); + assert.match(forced.stdout, /Generated codegen outputs\./); + assert.match((await generate("-trimprefix=Kind")).stdout, /Codegen outputs are up to date\./); fs.appendFileSync(source, "\nconst KindInvalid Kind = missing\n"); await assert.rejects(async () => await generate("-trimprefix=Kind")); fs.writeFileSync(source, "package probe\n\ntype Kind int\n\nconst KindFirst Kind = 0\n"); - assert.match((await generate("-trimprefix=Kind")).stdout, /Generated stringer output\./); - assert.match((await generate("-trimprefix=Kind")).stdout, /Stringer output is up to date\./); + assert.match((await generate("-trimprefix=Kind")).stdout, /Generated codegen outputs\./); + assert.match((await generate("-trimprefix=Kind")).stdout, /Codegen outputs are up to date\./); }); test("moq generation tracks interface inputs, output, and force", async context => { @@ -275,73 +302,91 @@ test("moq generation tracks interface inputs, output, and force", async context const output = path.join(directory, "mock/mock_generated.go"); fs.writeFileSync(base, "package probe\n\ntype Base interface { Read() string }\n"); fs.writeFileSync(source, "package probe\n\ntype Client interface { Base }\n"); - const command = [path.join(root, "tools/scripts/gen/generateMoq.mts"), "--input=client.go", "--input=base.go", "-stub", "-fmt=goimports", "-pkg=mock", "-out=mock/mock_generated.go"]; + const options = { + cwd: directory, + inputs: ["client.go", "base.go"], + outputs: ["mock/mock_generated.go"], + commands: [ + ["go", "tool", "github.com/matryer/moq", "-stub", "-fmt=goimports", "-pkg=mock", "-out=mock/mock_generated.go", ".", "Client"], + ["dprint", "fmt", "mock/mock_generated.go"], + ], + }; const generate = (...args: string[]) => - x(process.execPath, [...command, ...args, ".", "Client"], { + x(process.execPath, generatorArgs(options, args), { throwOnError: true, nodeOptions: { cwd: directory }, }); await generate(); const timestamp = fs.statSync(output).mtimeMs; - assert.match((await generate()).stdout, /Moq output is up to date\./); + assert.match((await generate()).stdout, /Codegen outputs are up to date\./); assert.equal(fs.statSync(output).mtimeMs, timestamp); fs.writeFileSync(base, "package probe\n\ntype Base interface { Read() string; Close() error }\n"); - assert.match((await generate()).stdout, /Generated moq output\./); + assert.match((await generate()).stdout, /Generated codegen outputs\./); assert.match(fs.readFileSync(output, "utf8"), /CloseFunc/); fs.rmSync(output); - assert.match((await generate()).stdout, /Generated moq output\./); - assert.match((await generate("--force")).stdout, /Generated moq output\./); - assert.match((await generate()).stdout, /Moq output is up to date\./); - const forced = await x(process.execPath, [...command, ".", "Client"], { + assert.match((await generate()).stdout, /Generated codegen outputs\./); + assert.match((await generate("--force")).stdout, /Generated codegen outputs\./); + assert.match((await generate()).stdout, /Codegen outputs are up to date\./); + const forced = await x(process.execPath, generatorArgs(options), { throwOnError: true, nodeOptions: { cwd: directory, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, }); - assert.match(forced.stdout, /Generated moq output\./); - assert.match((await generate()).stdout, /Moq output is up to date\./); + assert.match(forced.stdout, /Generated codegen outputs\./); + assert.match((await generate()).stdout, /Codegen outputs are up to date\./); }); test("diagnostic generation tracks Go and locale outputs and supports force", async context => { const root = path.resolve(import.meta.dirname, "../../.."); const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_diagnostics-probe-")); context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const output = path.join(directory, "diagnostics_generated.go"); + const localized = path.join(directory, "loc_generated.go"); + const options = { + cwd: path.join(root, "tsc/internal/diagnostics"), + inputs: ["generate.go", "diagnosticMessages.json", "extraDiagnosticMessages.json", "../{collections,json}/*.go", "../locale/lcl/*/diagnosticMessages/diagnosticMessages.generated.json.lcl"], + exclude: ["**/*_test.go"], + outputs: [output, localized, path.join(directory, "loc/*.json.gz")], + commands: [ + ["go", "run", "generate.go", "-diagnostics", output, "-loc", localized, "-locdir", path.join(directory, "loc")], + ["dprint", "fmt", output, localized], + ], + }; const generate = (...args: string[]) => - x(process.execPath, [path.join(root, "tools/scripts/gen/generateDiagnostics.mts"), "--outDir", directory, ...args], { + x(process.execPath, generatorArgs(options, args), { throwOnError: true, nodeOptions: { cwd: root }, }); await generate(); - const output = path.join(directory, "diagnostics_generated.go"); - const localized = path.join(directory, "loc_generated.go"); const locales = fs.globSync("loc/*.json.gz", { cwd: directory }).map(file => path.join(directory, file)); assert.ok(locales.length > 0); const files = [output, localized, ...locales]; const timestamps = files.map(file => fs.statSync(file).mtimeMs); - assert.match((await generate()).stdout, /Diagnostics are up to date\./); + assert.match((await generate()).stdout, /Codegen outputs are up to date\./); assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); const archive = fs.readFileSync(locales[0]); fs.rmSync(locales[0]); - assert.match((await generate()).stdout, /Generated diagnostics\./); + assert.match((await generate()).stdout, /Generated codegen outputs\./); assert.deepEqual(fs.readFileSync(locales[0]), archive); fs.writeFileSync(locales[0], "modified"); - assert.match((await generate()).stdout, /Generated diagnostics\./); + assert.match((await generate()).stdout, /Generated codegen outputs\./); assert.deepEqual(fs.readFileSync(locales[0]), archive); const unexpected = path.join(directory, "loc/unexpected.json.gz"); fs.writeFileSync(unexpected, "unexpected"); - assert.match((await generate()).stdout, /Generated diagnostics\./); + assert.match((await generate()).stdout, /Generated codegen outputs\./); assert.equal(fs.existsSync(unexpected), false); fs.rmSync(path.join(directory, "loc"), { recursive: true }); - assert.match((await generate()).stdout, /Generated diagnostics\./); + assert.match((await generate()).stdout, /Generated codegen outputs\./); assert.deepEqual(fs.readFileSync(locales[0]), archive); fs.rmSync(localized); - assert.match((await generate()).stdout, /Generated diagnostics\./); - assert.match((await generate("--force")).stdout, /Generated diagnostics\./); - assert.match((await generate()).stdout, /Diagnostics are up to date\./); - const forced = await x(process.execPath, [path.join(root, "tools/scripts/gen/generateDiagnostics.mts"), "--outDir", directory], { + assert.match((await generate()).stdout, /Generated codegen outputs\./); + assert.match((await generate("--force")).stdout, /Generated codegen outputs\./); + assert.match((await generate()).stdout, /Codegen outputs are up to date\./); + const forced = await x(process.execPath, generatorArgs(options), { throwOnError: true, nodeOptions: { cwd: root, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, }); - assert.match(forced.stdout, /Generated diagnostics\./); - assert.match((await generate()).stdout, /Diagnostics are up to date\./); + assert.match(forced.stdout, /Generated codegen outputs\./); + assert.match((await generate()).stdout, /Codegen outputs are up to date\./); }); test("Unicode generation skips unchanged tables and formatting", async () => { @@ -451,14 +496,14 @@ test("API protocol generation caches formatted output and supports force", async await generate(); const timestamp = fs.statSync(output).mtimeMs; const current = await generate(); - assert.match(current.stdout, /API protocol is up to date\./); - assert.doesNotMatch(current.stdout, /\$ node .*generateAPI\.mts/); + assert.match(current.stdout, /Codegen outputs are up to date\./); + assert.doesNotMatch(current.stdout, /\$ node .*cache\.mts/); assert.equal(fs.statSync(output).mtimeMs, timestamp); const unrelated = path.join(root, "tsc/internal/parser/codegen_cache_probe.go"); assert.equal(fs.existsSync(unrelated), false); context.after(() => fs.rmSync(unrelated, { force: true })); fs.writeFileSync(unrelated, "package parser\n"); - assert.match((await generate()).stdout, /API protocol is up to date\./); + assert.match((await generate()).stdout, /Codegen outputs are up to date\./); fs.rmSync(unrelated); const schema = path.join(root, "tsc/internal/api/requestfilesystem/codegen_cache_probe.go"); assert.equal(fs.existsSync(schema), false); @@ -470,19 +515,17 @@ test("API protocol generation caches formatted output and supports force", async } }); fs.writeFileSync(schema, 'package requestfilesystem\n\nconst KindCodegenCacheProbe Kind = "codegen-cache-probe"\n'); - assert.match((await generate()).stdout, /Generated API protocol\./); + assert.match((await generate()).stdout, /Generated codegen outputs\./); assert.match(fs.readFileSync(output, "utf8"), /"codegen-cache-probe"/); fs.rmSync(schema); - assert.match((await generate()).stdout, /Generated API protocol\./); + assert.match((await generate()).stdout, /Generated codegen outputs\./); assert.equal(fs.readFileSync(output, "utf8"), original); - assert.match((await generate(true)).stdout, /Generated API protocol\./); - assert.match((await generate()).stdout, /API protocol is up to date\./); - const direct = await x("node", ["./tools/scripts/gen/generateAPI.mts"], { throwOnError: true, nodeOptions: { cwd: root } }); - assert.match(direct.stdout, /API protocol is up to date\./); + assert.match((await generate(true)).stdout, /Generated codegen outputs\./); + assert.match((await generate()).stdout, /Codegen outputs are up to date\./); const nested = await x("go", ["-C", "./tsc", "generate", "./internal/api"], { throwOnError: true, nodeOptions: { cwd: root, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, }); - assert.match(nested.stdout, /Generated API protocol\./); - assert.match((await generate()).stdout, /API protocol is up to date\./); + assert.match(nested.stdout, /Generated codegen outputs\./); + assert.match((await generate()).stdout, /Codegen outputs are up to date\./); }); diff --git a/tools/scripts/gen/utils.mts b/tools/scripts/gen/utils.mts index 86f455a65f90d..af4bf23862a1c 100644 --- a/tools/scripts/gen/utils.mts +++ b/tools/scripts/gen/utils.mts @@ -19,32 +19,17 @@ export function goInputs(): string[] { return [import.meta.filename, ...globInputs(["go.work", "go.work.sum", "{tsc,tools}/go.{mod,sum}"])]; } -export function goEnvironment(): Record { - return Object.fromEntries(["GOOS", "GOARCH", "GOFLAGS", "GOTOOLCHAIN", "GOEXPERIMENT", "CGO_ENABLED", "GOWORK"].map(name => [name, process.env[name]])); -} - export function resolveForce(force?: boolean, environment = process.env.TSGO_HEREBY_FORCE): boolean { return force ?? /^(1|true|yes|on)$/i.test(environment ?? ""); } -export function parseGeneratorArgs(options: Options, args = process.argv.slice(2), goStyle = false) { - const { values, positionals, tokens } = parseArgs({ - args: goStyle ? args.map(arg => arg.replace(/^-(?=[a-z])/, "--")) : args, +export function parseGeneratorArgs(options: Options, args = process.argv.slice(2)) { + const { values } = parseArgs({ + args, options: { ...options, force: { type: "boolean" as const } }, - allowPositionals: goStyle, allowNegative: true, - tokens: true, - }); - const toolArgs = tokens.flatMap(token => { - if (token.kind === "positional") return [token.value]; - if (token.kind !== "option" || ["force", "no-force", "input"].includes(token.name)) return []; - return args.slice(token.index, token.index + (token.value !== undefined && !token.inlineValue ? 2 : 1)); }); - return { values, positionals, toolArgs, force: resolveForce((values as { force?: boolean; }).force) }; -} - -export async function runGo(args: string[], cwd = repoRoot): Promise { - await x("go", args, { throwOnError: true, nodeOptions: { cwd, stdio: "inherit" } }); + return { values, force: resolveForce((values as { force?: boolean; }).force) }; } export async function formatFiles(files: string[]): Promise { diff --git a/tools/scripts/tsc/generate-go-ast.ts b/tools/scripts/tsc/generate-go-ast.ts index 66235ca300424..7a85954a9836c 100644 --- a/tools/scripts/tsc/generate-go-ast.ts +++ b/tools/scripts/tsc/generate-go-ast.ts @@ -986,7 +986,7 @@ function generateKind(): string { w.write(""); w.write("package ast"); w.write(""); - w.write("//go:generate node ../../../tools/scripts/gen/generateStringer.mts -type=Kind -output=kind_stringer_generated.go"); + w.write("//go:generate npm run --silent cache -- --input $GOFILE --output kind_stringer_generated.go --command go tool golang.org/x/tools/cmd/stringer -type=Kind -output=kind_stringer_generated.go --command dprint fmt kind_stringer_generated.go"); w.write(""); w.write("type Kind int16"); w.write(""); diff --git a/tsc/internal/ast/kind_generated.go b/tsc/internal/ast/kind_generated.go index 5875ad539a876..46cf07a1d2dd1 100644 --- a/tsc/internal/ast/kind_generated.go +++ b/tsc/internal/ast/kind_generated.go @@ -2,7 +2,7 @@ package ast -//go:generate node ../../../tools/scripts/gen/generateStringer.mts -type=Kind -output=kind_stringer_generated.go +//go:generate npm run --silent cache -- --input $GOFILE --output kind_stringer_generated.go --command go tool golang.org/x/tools/cmd/stringer -type=Kind -output=kind_stringer_generated.go --command dprint fmt kind_stringer_generated.go type Kind int16 diff --git a/tsc/internal/bundled/bundled.go b/tsc/internal/bundled/bundled.go index b03040dc13e6c..c2f7d59c04f9a 100644 --- a/tsc/internal/bundled/bundled.go +++ b/tsc/internal/bundled/bundled.go @@ -11,7 +11,7 @@ import ( "github.com/microsoft/TypeScript/tsc/internal/vfs" ) -//go:generate node ../../../tools/scripts/gen/generateBundled.mts +//go:generate npm run --silent cache -- --input generate.go --input CopyrightNotice.txt --input "libs/*" --output libs_generated.go --output embed_generated.go --command go run generate.go // Define the below here to consolidate documentation. diff --git a/tsc/internal/checker/types.go b/tsc/internal/checker/types.go index fc8b44f1ca980..56891bb8804b7 100644 --- a/tsc/internal/checker/types.go +++ b/tsc/internal/checker/types.go @@ -11,7 +11,7 @@ import ( "github.com/microsoft/TypeScript/tsc/internal/evaluator" ) -//go:generate node ../../../tools/scripts/gen/generateStringer.mts -type=SignatureKind -output=stringer_generated.go +//go:generate npm run --silent cache -- --input $GOFILE --output stringer_generated.go --command go tool golang.org/x/tools/cmd/stringer -type=SignatureKind -output=stringer_generated.go --command dprint fmt stringer_generated.go // ParseFlags diff --git a/tsc/internal/core/compileroptions.go b/tsc/internal/core/compileroptions.go index 70953dd418c72..106343c2678b9 100644 --- a/tsc/internal/core/compileroptions.go +++ b/tsc/internal/core/compileroptions.go @@ -9,8 +9,8 @@ import ( "github.com/microsoft/TypeScript/tsc/internal/tspath" ) -//go:generate node ../../../tools/scripts/gen/generateStringer.mts -type=ModuleKind -trimprefix=ModuleKind -output=modulekind_stringer_generated.go -//go:generate node ../../../tools/scripts/gen/generateStringer.mts -type=ScriptTarget -trimprefix=ScriptTarget -output=scripttarget_stringer_generated.go +//go:generate npm run --silent cache -- --input $GOFILE --output modulekind_stringer_generated.go --command go tool golang.org/x/tools/cmd/stringer -type=ModuleKind -trimprefix=ModuleKind -output=modulekind_stringer_generated.go --command dprint fmt modulekind_stringer_generated.go +//go:generate npm run --silent cache -- --input $GOFILE --output scripttarget_stringer_generated.go --command go tool golang.org/x/tools/cmd/stringer -type=ScriptTarget -trimprefix=ScriptTarget -output=scripttarget_stringer_generated.go --command dprint fmt scripttarget_stringer_generated.go // CompilerOptions contains the compiler options exposed by the API. type CompilerOptions struct { diff --git a/tsc/internal/core/languagevariant.go b/tsc/internal/core/languagevariant.go index 49320bef85f24..83c3141aefd36 100644 --- a/tsc/internal/core/languagevariant.go +++ b/tsc/internal/core/languagevariant.go @@ -1,6 +1,6 @@ package core -//go:generate node ../../../tools/scripts/gen/generateStringer.mts -type=LanguageVariant -output=languagevariant_stringer_generated.go +//go:generate npm run --silent cache -- --input $GOFILE --output languagevariant_stringer_generated.go --command go tool golang.org/x/tools/cmd/stringer -type=LanguageVariant -output=languagevariant_stringer_generated.go --command dprint fmt languagevariant_stringer_generated.go type LanguageVariant int32 diff --git a/tsc/internal/core/scriptkind.go b/tsc/internal/core/scriptkind.go index 0f04a901d45a4..1813d9b5fe70d 100644 --- a/tsc/internal/core/scriptkind.go +++ b/tsc/internal/core/scriptkind.go @@ -1,6 +1,6 @@ package core -//go:generate node ../../../tools/scripts/gen/generateStringer.mts -type=ScriptKind -output=scriptkind_stringer_generated.go +//go:generate npm run --silent cache -- --input $GOFILE --output scriptkind_stringer_generated.go --command go tool golang.org/x/tools/cmd/stringer -type=ScriptKind -output=scriptkind_stringer_generated.go --command dprint fmt scriptkind_stringer_generated.go type ScriptKind int32 diff --git a/tsc/internal/core/tristate.go b/tsc/internal/core/tristate.go index 005ecfe086504..4c6b4032ae98b 100644 --- a/tsc/internal/core/tristate.go +++ b/tsc/internal/core/tristate.go @@ -1,6 +1,6 @@ package core -//go:generate node ../../../tools/scripts/gen/generateStringer.mts -type=Tristate -output=tristate_stringer_generated.go +//go:generate npm run --silent cache -- --input $GOFILE --output tristate_stringer_generated.go --command go tool golang.org/x/tools/cmd/stringer -type=Tristate -output=tristate_stringer_generated.go --command dprint fmt tristate_stringer_generated.go // Tristate diff --git a/tsc/internal/diagnostics/diagnostics.go b/tsc/internal/diagnostics/diagnostics.go index 3b2be93991d2a..2b0e01de4f76f 100644 --- a/tsc/internal/diagnostics/diagnostics.go +++ b/tsc/internal/diagnostics/diagnostics.go @@ -13,8 +13,8 @@ import ( "golang.org/x/text/language" ) -//go:generate node ../../../tools/scripts/gen/generateDiagnostics.mts -//go:generate node ../../../tools/scripts/gen/generateStringer.mts -type=Category -output=stringer_generated.go +//go:generate npm run --silent cache -- --input generate.go --input diagnosticMessages.json --input extraDiagnosticMessages.json --input "../{collections,json}/*.go" --input "../locale/lcl/*/diagnosticMessages/diagnosticMessages.generated.json.lcl" --exclude "**/*_test.go" --output diagnostics_generated.go --output loc_generated.go --output "loc/*.json.gz" --command go run generate.go -diagnostics diagnostics_generated.go -loc loc_generated.go -locdir loc --command dprint fmt diagnostics_generated.go loc_generated.go +//go:generate npm run --silent cache -- --input $GOFILE --output stringer_generated.go --command go tool golang.org/x/tools/cmd/stringer -type=Category -output=stringer_generated.go --command dprint fmt stringer_generated.go type Category int32 diff --git a/tsc/internal/ls/autoimport/export.go b/tsc/internal/ls/autoimport/export.go index 294136e7b48f9..d592ac2528977 100644 --- a/tsc/internal/ls/autoimport/export.go +++ b/tsc/internal/ls/autoimport/export.go @@ -8,7 +8,7 @@ import ( "github.com/microsoft/TypeScript/tsc/internal/tspath" ) -//go:generate node ../../../../tools/scripts/gen/generateStringer.mts -type=ExportSyntax -output=export_stringer_generated.go +//go:generate npm run --silent cache -- --input $GOFILE --output export_stringer_generated.go --command go tool golang.org/x/tools/cmd/stringer -type=ExportSyntax -output=export_stringer_generated.go --command dprint fmt export_stringer_generated.go // ModuleID uniquely identifies a module across multiple declarations. // If the export is from an ambient module declaration, this is the module name. diff --git a/tsc/internal/project/project.go b/tsc/internal/project/project.go index ea3a8fcc35740..6e438ca1eba5a 100644 --- a/tsc/internal/project/project.go +++ b/tsc/internal/project/project.go @@ -40,7 +40,7 @@ func SyntheticProgramID(path tspath.Path) (int, bool) { return id, err == nil && id > 0 } -//go:generate node ../../../tools/scripts/gen/generateStringer.mts -type=Kind -trimprefix=Kind -output=project_stringer_generated.go +//go:generate npm run --silent cache -- --input $GOFILE --output project_stringer_generated.go --command go tool golang.org/x/tools/cmd/stringer -type=Kind -trimprefix=Kind -output=project_stringer_generated.go --command dprint fmt project_stringer_generated.go type Kind int diff --git a/tsc/internal/testutil/projecttestutil/projecttestutil.go b/tsc/internal/testutil/projecttestutil/projecttestutil.go index bcd6f02042966..396ec6501f43a 100644 --- a/tsc/internal/testutil/projecttestutil/projecttestutil.go +++ b/tsc/internal/testutil/projecttestutil/projecttestutil.go @@ -23,9 +23,9 @@ import ( "github.com/microsoft/TypeScript/tsc/internal/vfs/vfstest" ) -//go:generate node ../../../../tools/scripts/gen/generateMoq.mts --input=../../project/client.go -stub -fmt goimports -pkg projecttestutil -out clientmock_generated.go ../../project Client +//go:generate npm run --silent cache -- --input $GOFILE --input ../../project/client.go --output clientmock_generated.go --command go tool github.com/matryer/moq -stub -fmt goimports -pkg projecttestutil -out clientmock_generated.go ../../project Client --command dprint fmt clientmock_generated.go -//go:generate node ../../../../tools/scripts/gen/generateMoq.mts --input=../../project/ata/ata.go -stub -fmt goimports -pkg projecttestutil -out npmexecutormock_generated.go ../../project/ata NpmExecutor +//go:generate npm run --silent cache -- --input $GOFILE --input ../../project/ata/ata.go --output npmexecutormock_generated.go --command go tool github.com/matryer/moq -stub -fmt goimports -pkg projecttestutil -out npmexecutormock_generated.go ../../project/ata NpmExecutor --command dprint fmt npmexecutormock_generated.go const ( TestTypingsLocation = "/home/src/Library/Caches/typescript" diff --git a/tsc/internal/vfs/vfs.go b/tsc/internal/vfs/vfs.go index 2a7000cb7c828..f32f96e5aefcf 100644 --- a/tsc/internal/vfs/vfs.go +++ b/tsc/internal/vfs/vfs.go @@ -5,7 +5,7 @@ import ( "time" ) -//go:generate node ../../../tools/scripts/gen/generateMoq.mts --input=vfs.go -fmt goimports -out vfsmock/mock_generated.go -pkg vfsmock . FS +//go:generate npm run --silent cache -- --input $GOFILE --output vfsmock/mock_generated.go --command go tool github.com/matryer/moq -fmt goimports -out vfsmock/mock_generated.go -pkg vfsmock . FS --command dprint fmt vfsmock/mock_generated.go // FS is a file system abstraction. type FS interface { diff --git a/tsc/internal/vfs/vfsmatch/vfsmatch.go b/tsc/internal/vfs/vfsmatch/vfsmatch.go index 6abe2fcec0133..cf118a466249a 100644 --- a/tsc/internal/vfs/vfsmatch/vfsmatch.go +++ b/tsc/internal/vfs/vfsmatch/vfsmatch.go @@ -12,7 +12,7 @@ import ( "github.com/microsoft/TypeScript/tsc/internal/vfs" ) -//go:generate node ../../../../tools/scripts/gen/generateStringer.mts -type=Usage -trimprefix=Usage -output=stringer_generated.go +//go:generate npm run --silent cache -- --input $GOFILE --output stringer_generated.go --command go tool golang.org/x/tools/cmd/stringer -type=Usage -trimprefix=Usage -output=stringer_generated.go --command dprint fmt stringer_generated.go // This file implements the glob matching algorithm specified in MATCHING_ALGORITHM.md. From 4603f1bf140ad25010431d605eb620f1dbc0c54c Mon Sep 17 00:00:00 2001 From: Wesley Wigham Date: Mon, 21 Sep 2026 10:15:09 -0700 Subject: [PATCH 4/6] Make herebyfile responsible for defining and offloading all codegen tasks - go:generate directives only for compat --- CONTRIBUTING.md | 7 + Herebyfile.mjs | 310 ++++++++++++++++-- package.json | 1 - tools/scripts/gen/cache.mts | 29 +- tools/scripts/gen/cache.test.mts | 148 ++++----- tools/scripts/gen/generatedFile.test.mts | 281 +++++++++------- tools/scripts/tsc/generate-go-ast.ts | 2 +- tsc/internal/ast/kind_generated.go | 2 +- tsc/internal/bundled/bundled.go | 2 +- tsc/internal/checker/types.go | 2 +- tsc/internal/core/compileroptions.go | 3 +- tsc/internal/core/languagevariant.go | 2 +- tsc/internal/core/scriptkind.go | 2 +- tsc/internal/core/tristate.go | 2 +- tsc/internal/diagnostics/diagnostics.go | 3 +- tsc/internal/ls/autoimport/export.go | 2 +- .../lsp/lsproto/_generate/fetchModel.mts | 57 ++-- .../lsp/lsproto/_generate/generate.mts | 9 +- tsc/internal/project/project.go | 2 +- tsc/internal/stringutil/generate.go | 2 +- .../projecttestutil/projecttestutil.go | 4 +- tsc/internal/vfs/vfs.go | 2 +- tsc/internal/vfs/vfsmatch/vfsmatch.go | 2 +- 23 files changed, 581 insertions(+), 295 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6c6d1776bffde..8d44df486ca45 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -104,11 +104,18 @@ npx hereby test # Run compiler and language-service Go tests npx hereby test:all # Also run benchmarks, tools, and API tests npx hereby lint # Run custom golangci-lint for both Go modules npx hereby generate # Regenerate compiler sources and bundled assets +npx hereby generate:all # Also regenerate AST, LSP, APIs, localization, and vendored files npx hereby format # Format Go, TypeScript, JSON, and YAML npx hereby check:format # Check formatting without changing files npx hereby tidy # Tidy both modules and synchronize go.work ``` +Generation is owned by `Herebyfile.mjs`; Go directives forward to the same +`generate:*` tasks for compatibility. Use a subtask such as +`npx hereby generate:diagnostics` to run one generator group, and pass `--force` +to bypass incremental caches. `generate:all` fetches the pinned LSP model when +its local cache is missing or stale. + Package-specific commands: ```bash diff --git a/Herebyfile.mjs b/Herebyfile.mjs index 82355c02b0491..8a2f6d2e043a0 100644 --- a/Herebyfile.mjs +++ b/Herebyfile.mjs @@ -369,19 +369,205 @@ export const cleanBuilt = task({ run: () => rimraf("built"), }); +/** @type {(() => Promise)[]} */ +const goGenerateActions = []; + async function runGenerate() { - return await run("go", ["generate", "-v", "./..."], { - cwd: "./tsc", - env: { TSGO_HEREBY_FORCE: options.force ? "1" : "0" }, - }); + for (const generate of goGenerateActions) { + await generate(); + } } export const generate = task({ name: "generate", - description: "Runs go generate on the project.", + description: "Runs the project's Go code generators directly. Pass --force to regenerate unchanged files.", run: runGenerate, }); +const getGoGenerateEnvironment = memoize(async () => { + const { stdout } = await runOutput("go", ["env", "-json", "GOOS", "GOARCH", "GOROOT"], { cwd: "./tsc" }); + return /** @type {{ GOOS: string; GOARCH: string; GOROOT: string }} */ (JSON.parse(stdout)); +}); + +/** @typedef {import("./tools/scripts/gen/cache.mts").CacheOptions & { file: string }} GoGenerator */ + +/** + * @param {string} name + * @param {GoGenerator} generator + */ +async function runGoGenerator(name, { file, ...spec }) { + const { default: cache } = await import("./tools/scripts/gen/cache.mts"); + const sourcePath = path.resolve(__dirname, file); + const source = fs.readFileSync(sourcePath, "utf8"); + const packageName = /^package\s+(\w+)/m.exec(source)?.[1]; + const line = source.split(/\r?\n/).findIndex(line => line.trimEnd() === `//go:generate npx hereby ${name}`) + 1; + assert(packageName && line, `Missing package or generation directive in ${file}`); + const goEnv = await getGoGenerateEnvironment(); + const pathKey = process.platform === "win32" ? Object.keys(process.env).find(key => key.toUpperCase() === "PATH") ?? "PATH" : "PATH"; + const goBin = path.join(goEnv.GOROOT, "bin"); + const searchPath = (process.env[pathKey] ?? "").split(path.delimiter).filter(entry => path.resolve(entry) !== goBin); + await cache({ + ...spec, + cwd: spec.cwd ?? path.dirname(sourcePath), + inputs: [__filename, sourcePath, ...spec.inputs], + envInputs: [ + ...(spec.envInputs ?? ["GOOS", "GOARCH", "GOFLAGS", "GOTOOLCHAIN", "GOEXPERIMENT", "CGO_ENABLED", "GOWORK"]), + "GOROOT", + "GOFILE", + "GOLINE", + "GOPACKAGE", + "DOLLAR", + ], + env: { + ...goEnv, + GOFILE: path.basename(sourcePath), + GOLINE: String(line), + GOPACKAGE: packageName, + DOLLAR: "$", + [pathKey]: [goBin, ...searchPath].join(path.delimiter), + }, + force: !!options.force, + }); +} + +/** + * @param {string} name + * @param {GoGenerator[] | (() => Promise)} generators + */ +function goGenerateTask(name, generators) { + const run = typeof generators === "function" ? generators : async () => { + for (const generator of generators) await runGoGenerator(name, generator); + }; + goGenerateActions.push(run); + return task({ + name, + description: `Generates ${name.slice("generate:".length)} files. Pass --force to regenerate unchanged files.`, + run, + }); +} + +/** + * @param {string} file + * @param {string} type + * @param {string} output + * @param {string} [trimPrefix] + * @returns {GoGenerator} + */ +function stringerGenerator(file, type, output, trimPrefix) { + return { + file, + inputs: [], + outputs: [output], + commands: [ + ["go", "tool", "golang.org/x/tools/cmd/stringer", `-type=${type}`, ...(trimPrefix ? [`-trimprefix=${trimPrefix}`] : []), `-output=${output}`], + ["dprint", "fmt", output], + ], + }; +} + +/** + * @param {string} file + * @param {string} type + * @param {string} output + * @param {{ source?: string; packageName: string; inputs?: string[]; stub?: boolean }} options + * @returns {GoGenerator} + */ +function moqGenerator(file, type, output, { source = ".", packageName, inputs = [], stub = false }) { + return { + file, + inputs, + outputs: [output], + commands: [ + ["go", "tool", "github.com/matryer/moq", ...(stub ? ["-stub"] : []), "-fmt", "goimports", "-pkg", packageName, "-out", output, source, type], + ["dprint", "fmt", output], + ], + }; +} + +async function runGenerateASTStringer() { + await runGoGenerator("generate:ast-stringer", stringerGenerator("tsc/internal/ast/kind_generated.go", "Kind", "kind_stringer_generated.go")); +} + +export const generateASTStringer = goGenerateTask("generate:ast-stringer", runGenerateASTStringer); + +export const generateBundled = goGenerateTask("generate:bundled", [{ + file: "tsc/internal/bundled/bundled.go", + inputs: ["generate.go", "CopyrightNotice.txt", "libs/*"], + outputs: ["libs_generated.go", "embed_generated.go"], + commands: [["go", "run", "generate.go"]], +}]); + +export const generateChecker = goGenerateTask("generate:checker", [ + stringerGenerator("tsc/internal/checker/types.go", "SignatureKind", "stringer_generated.go"), +]); + +export const generateCompilerOptions = goGenerateTask("generate:compileroptions", [ + stringerGenerator("tsc/internal/core/compileroptions.go", "ModuleKind", "modulekind_stringer_generated.go", "ModuleKind"), + stringerGenerator("tsc/internal/core/compileroptions.go", "ScriptTarget", "scripttarget_stringer_generated.go", "ScriptTarget"), +]); + +export const generateLanguageVariant = goGenerateTask("generate:languagevariant", [ + stringerGenerator("tsc/internal/core/languagevariant.go", "LanguageVariant", "languagevariant_stringer_generated.go"), +]); + +export const generateScriptKind = goGenerateTask("generate:scriptkind", [ + stringerGenerator("tsc/internal/core/scriptkind.go", "ScriptKind", "scriptkind_stringer_generated.go"), +]); + +export const generateTristate = goGenerateTask("generate:tristate", [ + stringerGenerator("tsc/internal/core/tristate.go", "Tristate", "tristate_stringer_generated.go"), +]); + +export const generateDiagnostics = goGenerateTask("generate:diagnostics", [ + { + file: "tsc/internal/diagnostics/diagnostics.go", + inputs: ["generate.go", "diagnosticMessages.json", "extraDiagnosticMessages.json", "../{collections,json}/*.go", "../locale/lcl/*/diagnosticMessages/diagnosticMessages.generated.json.lcl"], + exclude: ["**/*_test.go"], + outputs: ["diagnostics_generated.go", "loc_generated.go", "loc/*.json.gz"], + commands: [ + ["go", "run", "generate.go", "-diagnostics", "diagnostics_generated.go", "-loc", "loc_generated.go", "-locdir", "loc"], + ["dprint", "fmt", "diagnostics_generated.go", "loc_generated.go"], + ], + }, + stringerGenerator("tsc/internal/diagnostics/diagnostics.go", "Category", "stringer_generated.go"), +]); + +export const generateAutoImport = goGenerateTask("generate:autoimport", [ + stringerGenerator("tsc/internal/ls/autoimport/export.go", "ExportSyntax", "export_stringer_generated.go"), +]); + +export const generateProject = goGenerateTask("generate:project", [ + stringerGenerator("tsc/internal/project/project.go", "Kind", "project_stringer_generated.go", "Kind"), +]); + +export const generateProjectTestUtil = goGenerateTask("generate:projecttestutil", [ + moqGenerator("tsc/internal/testutil/projecttestutil/projecttestutil.go", "Client", "clientmock_generated.go", { + source: "../../project", + packageName: "projecttestutil", + inputs: ["../../project/client.go"], + stub: true, + }), + moqGenerator("tsc/internal/testutil/projecttestutil/projecttestutil.go", "NpmExecutor", "npmexecutormock_generated.go", { + source: "../../project/ata", + packageName: "projecttestutil", + inputs: ["../../project/ata/ata.go"], + stub: true, + }), +]); + +export const generateVFS = goGenerateTask("generate:vfs", [ + moqGenerator("tsc/internal/vfs/vfs.go", "FS", "vfsmock/mock_generated.go", { packageName: "vfsmock" }), +]); + +export const generateVFSMatch = goGenerateTask("generate:vfsmatch", [ + stringerGenerator("tsc/internal/vfs/vfsmatch/vfsmatch.go", "Usage", "stringer_generated.go", "Usage"), +]); + +export const generateUnicode = goGenerateTask("generate:unicode", async () => { + const { default: generate } = await import("./tsc/internal/stringutil/_scripts/generate-unicode-data.mts"); + await generate(!!options.force); +}); + async function runGenerateExtension() { return await run("npm", ["run", "-w", "native-preview", "generateLocBundle"]); } @@ -392,6 +578,48 @@ export const generateExtension = task({ run: runGenerateExtension, }); +async function runGenerateExtensionTest() { + await run("npm", ["run", "-w", "native-preview", "generateLocTest"]); +} + +export const generateExtensionTest = task({ + name: "generate:extension-test", + description: "Generates pseudo-localized extension resources.", + dependencies: [generateExtension], + run: runGenerateExtensionTest, +}); + +async function runGenerateLSP() { + const { GeneratedFile } = await import("./tools/scripts/gen/generatedFile.mts"); + const directory = path.join(__dirname, "tsc/internal/lsp/lsproto/_generate"); + const modelFiles = ["metaModel.json", "metaModelSchema.mts"].map(file => new GeneratedFile(path.join(directory, file), [path.join(directory, "fetchModel.mts"), path.join(__dirname, "package-lock.json")])); + if (!modelFiles.every(file => file.isCurrent(!!options.force))) { + for (const file of modelFiles) file.invalidate(); + const { default: fetchModel } = await import("./tsc/internal/lsp/lsproto/_generate/fetchModel.mts"); + await fetchModel(); + for (const file of modelFiles) file.markCurrent(); + } + const output = new GeneratedFile(path.join(directory, "../lsp_generated.go"), [ + __filename, + path.join(directory, "generate.mts"), + ...modelFiles.map(file => file.fileName), + ]); + if (output.isCurrent(!!options.force)) { + console.log("LSP bindings are up to date."); + return; + } + output.invalidate(); + const { default: generate } = await import("./tsc/internal/lsp/lsproto/_generate/generate.mts"); + await generate(); + output.markCurrent(); +} + +export const generateLSP = task({ + name: "generate:lsp", + description: "Generates LSP bindings from the pinned protocol model. Pass --force to regenerate unchanged files.", + run: runGenerateLSP, +}); + // ── Enum generation from Go source ────────────────────────────── /** @@ -988,21 +1216,34 @@ export const generateEnums = task({ run: runGenerateEnums, }); +async function runGenerateAST() { + const { default: generate } = await import("./tools/scripts/tsc/generate.ts"); + generate(!!options.force); + await runGenerateASTStringer(); +} + export const generateAST = task({ name: "generate:ast", - description: "Generates AST and encoder files from ast.json. Pass --force to regenerate unchanged files.", - run: async () => { - const { default: generate } = await import("./tools/scripts/tsc/generate.ts"); - generate(!!options.force); - }, + description: "Generates AST, kind stringer, and encoder files from ast.json. Pass --force to regenerate unchanged files.", + run: runGenerateAST, +}); + +async function runGenerateSync() { + const { generateSync } = await import("./packages/typescript/scripts/generateSync.ts"); + generateSync(!!options.force); +} + +export const generateSync = task({ + name: "generate:sync", + description: "Generates synchronous and generator APIs and tests. Pass --force to regenerate unchanged files.", + run: runGenerateSync, }); async function runGenerateAPI() { - const { default: cache } = await import("./tools/scripts/gen/cache.mts"); - await cache({ + await runGoGenerator("generate:api", { + file: "tsc/internal/api/proto.go", cwd: __dirname, inputs: [ - __filename, "tsc/internal/api/*.go", "tsc/internal/api/requestfilesystem/*.go", "tsc/internal/core/*.go", @@ -1018,15 +1259,10 @@ async function runGenerateAPI() { ["go", "-C", "./tools", "run", "./gen-proto", "../tsc/internal/api/proto.go", "../packages/typescript/src/api/proto.generated.ts"], ["dprint", "fmt", "packages/typescript/src/api/proto.generated.ts"], ], - force: !!options.force, }); } -export const generateAPI = task({ - name: "generate:api", - description: "Generates API files from internal/api/proto.go and internal/api/session.go. Pass --force to regenerate unchanged files.", - run: runGenerateAPI, -}); +export const generateAPI = goGenerateTask("generate:api", runGenerateAPI); // ── Vendored npm dependencies ─────────────────────────────────── @@ -1056,6 +1292,22 @@ export const generateVendor = task({ run: runGenerateVendor, }); +const generateAllCompiler = task({ + name: "generate:all:compiler", + hiddenFromTaskList: true, + dependencies: [generateAST, generateLSP], + run: async () => { + await runGenerate(); + await runGenerateEnums(); + }, +}); + +export const generateAll = task({ + name: "generate:all", + description: "Runs all code generation, including AST, LSP, APIs, extension localization, and vendored dependencies.", + dependencies: [generateAllCompiler, generateSync, generateExtensionTest, generateVendor], +}); + const coverageDir = path.join(__dirname, "coverage"); const ensureCoverageDirExists = memoize(() => { @@ -1293,9 +1545,8 @@ export const buildAPI = task({ }, }); -async function runBuildAPITests() { - const { generateSync } = await import("./packages/typescript/scripts/generateSync.ts"); - generateSync(!!options.force); +async function runBuildAPITests(generateSources = true) { + if (generateSources) await runGenerateSync(); await run("npm", ["run", "-w", "@typescript/typescript", "build:test"]); } @@ -1452,9 +1703,12 @@ async function runFormat() { export const validate = task({ name: "validate", - description: "Builds, tests, lints, and formats the repo. Pass --api to include API tests, or --all to include all ancilliary repository tests.", - dependencies: [build], + description: "Generates, builds, tests, lints, and formats the repo. Pass --api to include API tests, or --all to include all code generation and ancillary repository tests.", + dependencies: [options.all ? generateAll : generate], run: async () => { + await generateLibs(builtLocal); + await buildTsc({ extraFlags: options.release ? getReleaseBuildFlags() : [] }); + /** @type {{ name: string; error: unknown }[]} */ const failures = []; /** @param {string} name @param {() => Promise} action */ @@ -1468,18 +1722,14 @@ export const validate = task({ } }; - await runGenerate(); await runValidation("test:tsc", runTests); await runValidation("test:extension", runTestExtension); if (options.api || options.all) { - await runGenerateEnums(); // prereqs for test:api not included in `validate` deps - await runGenerateAPI(); - await runBuildAPITests(); + if (!options.all) await runGenerateEnums(); + await runBuildAPITests(!options.all); await runValidation("test:api", runTestAPI); } if (options.all) { - await runGenerateExtension(); - await runGenerateVendor(); await runValidation("test:benchmarks", runTestBenchmarks); await runValidation("test:tools", runTestTools); await runValidation("test:smoke", runSmokeTest); // in CI this is run with `--race` diff --git a/package.json b/package.json index fc7edb42f8bb8..fbdcc207eed23 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,6 @@ "format": "hereby format", "check:format": "hereby check:format", "generate": "hereby generate", - "cache": "node tools/scripts/gen/cache.mts", "tidy": "hereby tidy", "extension:build": "npm run -w native-preview build", "extension:watch": "npm run -w native-preview watch", diff --git a/tools/scripts/gen/cache.mts b/tools/scripts/gen/cache.mts index 44241d8d248f9..f309af3b0c8c5 100644 --- a/tools/scripts/gen/cache.mts +++ b/tools/scripts/gen/cache.mts @@ -5,7 +5,6 @@ import { x } from "tinyexec"; import { GeneratedFile } from "./generatedFile.mts"; import { goInputs, - parseGeneratorArgs, repoRoot, } from "./utils.mts"; @@ -16,6 +15,7 @@ export interface CacheOptions { cwd?: string; exclude?: string[]; envInputs?: string[]; + env?: NodeJS.ProcessEnv; force?: boolean; } @@ -30,12 +30,14 @@ export default async function cache({ cwd = process.cwd(), exclude = [], envInputs = ["GOOS", "GOARCH", "GOFLAGS", "GOTOOLCHAIN", "GOEXPERIMENT", "CGO_ENABLED", "GOWORK"], + env, force = false, }: CacheOptions): Promise { if (!outputs.length || !commands.length || commands.some(command => !command.length)) { throw new Error("Cached generation requires outputs and nonempty commands."); } cwd = path.resolve(cwd); + const environment = { ...process.env, ...env }; const snapshotInputs = () => { const entries = [ ...new Set([ @@ -76,7 +78,7 @@ export default async function cache({ exclude, inputHash: before.hash, outputFiles: files, - environment: envInputs.map(name => [name, process.env[name]]), + environment: envInputs.map(name => [name, environment[name]]), }) ); const previous = artifacts(outputFiles()); @@ -88,7 +90,7 @@ export default async function cache({ for (const file of previous) file.invalidate(); for (const [command, ...args] of commands) { - await x(command, args, { throwOnError: true, nodeOptions: { cwd, stdio: "inherit" } }); + await x(command, args, { throwOnError: true, nodeOptions: { cwd, env: environment, stdio: "inherit" } }); } if (!complete()) throw new Error(`Generation did not produce all declared outputs: ${outputs.join(", ")}`); if (snapshotInputs().hash === before.hash) { @@ -97,24 +99,3 @@ export default async function cache({ console.log("Generated codegen outputs."); return false; } - -if (process.argv[1] === import.meta.filename) { - const args = process.argv.slice(2); - const cwd = process.env.npm_lifecycle_event === "cache" ? process.env.INIT_CWD ?? process.cwd() : process.cwd(); - const commandStart = args.indexOf("--command"); - const { values, force } = parseGeneratorArgs({ - input: { type: "string", multiple: true }, - output: { type: "string", multiple: true }, - exclude: { type: "string", multiple: true }, - env: { type: "string", multiple: true }, - cwd: { type: "string" }, - }, commandStart < 0 ? args : args.slice(0, commandStart)); - const commands: string[][] = []; - if (commandStart >= 0) { - for (const arg of args.slice(commandStart)) { - if (arg === "--command") commands.push([]); - else commands[commands.length - 1].push(arg); - } - } - await cache({ inputs: values.input ?? [], outputs: values.output ?? [], commands, cwd: path.resolve(cwd, values.cwd ?? "."), exclude: values.exclude, envInputs: values.env, force }); -} diff --git a/tools/scripts/gen/cache.test.mts b/tools/scripts/gen/cache.test.mts index 83e7426f309ff..aeb4445078a4f 100644 --- a/tools/scripts/gen/cache.test.mts +++ b/tools/scripts/gen/cache.test.mts @@ -3,52 +3,59 @@ import * as fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { test } from "node:test"; -import { x } from "tinyexec"; import cache from "./cache.mts"; -test("npm cache preserves the caller directory and command arguments without task logging", async context => { +test("cache commands use and track an explicit environment without changing the parent", async context => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-cache-env-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const parentValue = process.env.GOFILE; + const options = { + cwd: directory, + inputs: [], + outputs: ["output.json"], + envInputs: ["GOFILE", "GOPACKAGE"], + env: { GOFILE: "source.go", GOPACKAGE: "example" }, + commands: [[process.execPath, "-e", "require('node:fs').writeFileSync('output.json', JSON.stringify({ file: process.env.GOFILE, pkg: process.env.GOPACKAGE, cwd: process.cwd() }))"]], + }; + assert.equal(await cache(options), false); + assert.deepEqual(JSON.parse(fs.readFileSync(path.join(directory, "output.json"), "utf8")), { + file: "source.go", + pkg: "example", + cwd: directory, + }); + assert.equal(await cache(options), true); + assert.equal(await cache({ ...options, env: { ...options.env, GOFILE: "other.go" } }), false); + assert.equal(process.env.GOFILE, parentValue); +}); + +test("cache commands preserve the working directory and arguments", async context => { const root = path.resolve(import.meta.dirname, "../../.."); const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_cache probe-")); context.after(() => fs.rmSync(directory, { recursive: true, force: true, maxRetries: 10 })); fs.writeFileSync(path.join(directory, "input.txt"), "input"); fs.writeFileSync(path.join(directory, "generate.cjs"), "const fs = require('node:fs'); fs.writeFileSync('output.txt', fs.readFileSync('input.txt') + JSON.stringify(process.argv.slice(2)));\n"); fs.writeFileSync(path.join(directory, "finish.cjs"), "require('node:fs').appendFileSync('output.txt', ' finished');\n"); - const generate = (flags: string[] = [], env = process.env, failure = false) => - x("npm", [ - "run", - "--silent", - "cache", - "--", - "--input", - "input.txt", - "--input", - "*.cjs", - "--output", - "output.txt", - ...flags, - "--command", - process.execPath, - "generate.cjs", - "--help", - "--force", - "space value", - "--command", - process.execPath, - failure ? "missing.cjs" : "finish.cjs", - ], { throwOnError: true, nodeOptions: { cwd: directory, env: { ...env, INIT_CWD: root } } }); + const generate = (force = false, failure = false) => + cache({ + cwd: directory, + inputs: ["input.txt", "*.cjs"], + outputs: ["output.txt"], + commands: [ + [process.execPath, "generate.cjs", "--help", "--force", "space value"], + [process.execPath, failure ? "missing.cjs" : "finish.cjs"], + ], + force, + }); const output = path.join(directory, "output.txt"); - assert.equal((await generate()).stdout.trim(), "Generated codegen outputs."); + assert.equal(await generate(), false); assert.equal(fs.readFileSync(output, "utf8"), 'input["--help","--force","space value"] finished'); const timestamp = fs.statSync(output).mtimeMs; - assert.equal((await generate()).stdout.trim(), "Codegen outputs are up to date."); + assert.equal(await generate(), true); assert.equal(fs.statSync(output).mtimeMs, timestamp); - assert.doesNotMatch((await generate(["--force"])).stdout, /up to date/); - const forced = { ...process.env, TSGO_HEREBY_FORCE: "1" }; - assert.match((await generate(["--no-force"], forced)).stdout, /up to date/); - assert.doesNotMatch((await generate([], forced)).stdout, /up to date/); - await assert.rejects(async () => await generate([], process.env, true)); - assert.doesNotMatch((await generate()).stdout, /up to date/); - assert.match((await generate()).stdout, /up to date/); + assert.equal(await generate(true), false); + await assert.rejects(generate(false, true)); + assert.equal(await generate(), false); + assert.equal(await generate(), true); }); test("cache commands track inputs, outputs, arguments, and failures", async context => { @@ -59,64 +66,51 @@ test("cache commands track inputs, outputs, arguments, and failures", async cont fs.writeFileSync(input, "first"); fs.writeFileSync(path.join(directory, "generate.cjs"), "const fs = require('node:fs'); fs.writeFileSync('output.txt', fs.readFileSync('input.txt'));\n"); fs.writeFileSync(path.join(directory, "finish.cjs"), "const fs = require('node:fs'); fs.appendFileSync('output.txt', process.argv[2]); fs.writeFileSync('extra.txt', 'extra');\n"); - const generate = (suffix = " done", flags: string[] = [], failure = false, env = process.env) => - x(process.execPath, [ - path.join(import.meta.dirname, "cache.mts"), - "--env", - "TSGO_CACHE_PROBE", - "--input", - "input*.txt", - "--input", - "*.cjs", - "--output", - "output.txt", - "--output", - "extra*.txt", - ...flags, - "--command", - process.execPath, - "generate.cjs", - "--command", - process.execPath, - failure ? "missing.cjs" : "finish.cjs", - suffix, - ], { throwOnError: true, nodeOptions: { cwd: directory, env } }); + const generate = (suffix = " done", force = false, failure = false, env: NodeJS.ProcessEnv = {}) => + cache({ + cwd: directory, + inputs: ["input*.txt", "*.cjs"], + outputs: ["output.txt", "extra*.txt"], + envInputs: ["TSGO_CACHE_PROBE"], + env, + commands: [ + [process.execPath, "generate.cjs"], + [process.execPath, failure ? "missing.cjs" : "finish.cjs", suffix], + ], + force, + }); await generate(); assert.equal(fs.readFileSync(output, "utf8"), "first done"); const timestamp = fs.statSync(output).mtimeMs; - assert.match((await generate()).stdout, /up to date/); + assert.equal(await generate(), true); assert.equal(fs.statSync(output).mtimeMs, timestamp); fs.writeFileSync(input, "second"); await generate(); assert.equal(fs.readFileSync(output, "utf8"), "second done"); const addedInput = path.join(directory, "input-added.txt"); fs.writeFileSync(addedInput, "added"); - assert.doesNotMatch((await generate()).stdout, /up to date/); + assert.equal(await generate(), false); fs.rmSync(addedInput); - assert.doesNotMatch((await generate()).stdout, /up to date/); + assert.equal(await generate(), false); fs.rmSync(path.join(directory, "extra.txt")); - assert.doesNotMatch((await generate()).stdout, /up to date/); + assert.equal(await generate(), false); fs.writeFileSync(path.join(directory, "extra-added.txt"), "extra"); - assert.doesNotMatch((await generate()).stdout, /up to date/); + assert.equal(await generate(), false); fs.writeFileSync(output, "edited"); - assert.doesNotMatch((await generate()).stdout, /up to date/); - assert.doesNotMatch((await generate(" changed")).stdout, /up to date/); + assert.equal(await generate(), false); + assert.equal(await generate(" changed"), false); assert.equal(fs.readFileSync(output, "utf8"), "second changed"); - assert.match((await generate(" changed")).stdout, /up to date/); - assert.doesNotMatch((await generate(" changed", ["--force"])).stdout, /up to date/); - await assert.rejects(async () => await generate(" changed", ["--force"], true)); - assert.doesNotMatch((await generate(" changed")).stdout, /up to date/); - assert.match((await generate(" changed")).stdout, /up to date/); - const forced = { ...process.env, TSGO_HEREBY_FORCE: "1" }; - assert.match((await generate(" changed", ["--no-force"], false, forced)).stdout, /up to date/); - assert.doesNotMatch((await generate(" changed", [], false, forced)).stdout, /up to date/); - assert.match((await generate(" changed")).stdout, /up to date/); - const environment = { ...process.env, TSGO_CACHE_PROBE: "changed" }; - assert.doesNotMatch((await generate(" changed", [], false, environment)).stdout, /up to date/); - assert.match((await generate(" changed", [], false, environment)).stdout, /up to date/); - assert.doesNotMatch((await generate("--force")).stdout, /up to date/); + assert.equal(await generate(" changed"), true); + assert.equal(await generate(" changed", true), false); + await assert.rejects(generate(" changed", true, true)); + assert.equal(await generate(" changed"), false); + assert.equal(await generate(" changed"), true); + const environment = { TSGO_CACHE_PROBE: "changed" }; + assert.equal(await generate(" changed", false, false, environment), false); + assert.equal(await generate(" changed", false, false, environment), true); + assert.equal(await generate("--force"), false); assert.equal(fs.readFileSync(output, "utf8"), "second--force"); - assert.match((await generate("--force")).stdout, /up to date/); + assert.equal(await generate("--force"), true); }); test("cache commands reject missing inputs, outputs, and empty commands", async context => { diff --git a/tools/scripts/gen/generatedFile.test.mts b/tools/scripts/gen/generatedFile.test.mts index 881111faf8c03..6eb0c0f499c6d 100644 --- a/tools/scripts/gen/generatedFile.test.mts +++ b/tools/scripts/gen/generatedFile.test.mts @@ -4,22 +4,68 @@ import * as fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { test } from "node:test"; +import { stripVTControlCharacters } from "node:util"; +import { runInNewContext } from "node:vm"; import { x } from "tinyexec"; -import type { CacheOptions } from "./cache.mts"; +import ts from "typescript"; +import cache from "./cache.mts"; import { GeneratedFile } from "./generatedFile.mts"; -function generatorArgs(options: CacheOptions, flags: string[] = []): string[] { - return [ - path.join(import.meta.dirname, "cache.mts"), - ...(options.cwd ? ["--cwd", options.cwd] : []), - ...options.inputs.flatMap(file => ["--input", file]), - ...options.outputs.flatMap(file => ["--output", file]), - ...(options.exclude ?? []).flatMap(pattern => ["--exclude", pattern]), - ...(options.envInputs ?? []).flatMap(name => ["--env", name]), - ...flags, - ...options.commands.flatMap(command => ["--command", ...command]), - ]; -} +test("validate generates before building and reuses generate:all", async () => { + const fileName = path.resolve(import.meta.dirname, "../../../Herebyfile.mjs"); + const source = ts.createSourceFile(fileName, fs.readFileSync(fileName, "utf8"), ts.ScriptTarget.Latest, true, ts.ScriptKind.JS); + const declaration = source.statements.filter(ts.isVariableStatement) + .flatMap(statement => statement.declarationList.declarations) + .find(declaration => declaration.name.getText(source) === "validate"); + assert.ok(declaration?.initializer); + for (const options of [{}, { api: true }, { all: true }]) { + const calls: string[] = []; + const action = (name: string) => async () => { + calls.push(name); + }; + const generate = { run: action("generate") }; + const generateAll = { run: action("generate:all") }; + const build = { run: action("build") }; + const validation = runInNewContext(declaration.initializer.getText(source), { + task: (spec: unknown) => spec, + options, + generate, + generateAll, + build, + builtLocal: "./built/local", + generateLibs: action("lib"), + buildTsc: action("build"), + getReleaseBuildFlags: () => [], + runGenerate: action("generate"), + runGenerateEnums: action("generate:enums"), + runGenerateAPI: action("generate:api"), + runGenerateExtension: action("generate:extension"), + runGenerateVendor: action("generate:vendor"), + runBuildAPITests: async (generate = true) => { + if (generate) calls.push("generate:sync"); + calls.push("build:api:test"); + }, + runTests: action("test:tsc"), + runTestExtension: action("test:extension"), + runTestAPI: action("test:api"), + runTestBenchmarks: action("test:benchmarks"), + runTestTools: action("test:tools"), + runSmokeTest: action("test:smoke"), + runLint: action("lint"), + runFormat: action("format"), + }) as { dependencies: { run: () => Promise; }[]; run: () => Promise; }; + await Promise.all(validation.dependencies.map(dependency => dependency.run())); + await validation.run(); + assert.equal(calls[0], "all" in options ? "generate:all" : "generate"); + assert.ok(calls.indexOf("build") > 0); + assert.ok(calls.indexOf("build") < calls.indexOf("test:tsc")); + assert.equal(calls.includes("test:api"), "api" in options || "all" in options); + assert.equal(calls.includes("test:tools"), "all" in options); + if ("all" in options) { + assert.deepEqual(calls.filter(name => name.startsWith("generate")), ["generate:all"]); + } + } +}); test("generator options preserve repeated inputs and force overrides", async () => { const { parseGeneratorArgs, resolveForce } = await import("./utils.mts"); @@ -147,6 +193,57 @@ test("generated files track each output independently", context => { assert.equal(files[1].isCurrent(), false); }); +test("Hereby generation runs Go generators directly and shares caches with Go fallback", async () => { + const root = path.resolve(import.meta.dirname, "../../.."); + const generate = () => x("npx", ["hereby", "generate"], { throwOnError: true, nodeOptions: { cwd: root } }); + const first = await generate(); + assert.doesNotMatch(first.stdout, /\$ go generate|npm run --silent cache|\$ node .*generate-unicode-data/); + const files = fs.globSync(["tsc/internal/**/*generated.go", "packages/typescript/src/api/proto.generated.ts"], { cwd: root }); + const timestamps = files.map(file => fs.statSync(path.join(root, file)).mtimeMs); + const current = await generate(); + assert.doesNotMatch(current.stdout, /Generated codegen outputs|Generated Unicode tables/); + assert.match(current.stdout, /Unicode tables are up to date/); + assert.deepEqual(files.map(file => fs.statSync(path.join(root, file)).mtimeMs), timestamps); + const fallback = await x("go", ["-C", "./tsc", "generate", "./internal/diagnostics"], { throwOnError: true, nodeOptions: { cwd: root } }); + assert.equal(fallback.stdout.match(/Codegen outputs are up to date/g)?.length, 2); + const nested = await x("npx", ["hereby", "generate:compileroptions"], { throwOnError: true, nodeOptions: { cwd: path.join(root, "tsc/internal/core") } }); + assert.equal(nested.stdout.match(/Codegen outputs are up to date/g)?.length, 2); +}); + +test("generate:all includes standalone generators without Go traversal", async () => { + const root = path.resolve(import.meta.dirname, "../../.."); + const { stdout } = await x("npx", ["hereby", "generate:all"], { throwOnError: true, nodeOptions: { cwd: root } }); + const log = stripVTControlCharacters(stdout); + const eventIndex = (event: string) => { + const index = log.indexOf(event); + assert.ok(index >= 0, `Missing task event: ${event}`); + return index; + }; + const compilerStart = eventIndex("Starting generate:all:compiler"); + assert.ok(eventIndex("Finished generate:ast ") < compilerStart); + assert.ok(eventIndex("Finished generate:lsp ") < compilerStart); + assert.ok(eventIndex("Starting generate:sync") < compilerStart); + assert.ok(eventIndex("Starting generate:vendor") < compilerStart); + assert.ok(eventIndex("Finished generate:extension ") < eventIndex("Starting generate:extension-test")); + assert.doesNotMatch(stdout, /\$ go generate|npm run --silent cache/); + assert.match(stdout, /Unicode tables/); + assert.match(stdout, /Enums are up to date|All generated values match Go/); + assert.match(stdout, /Sync API is up to date|Formatting\.\.\./); + assert.match(stdout, /LSP bindings are up to date|Successfully generated .*lsp_generated\.go/); + assert.match(stdout, /generateLocBundle/); + assert.match(stdout, /generateLocTest/); + assert.deepEqual( + fs.readFileSync(path.join(root, "packages/typescript/vendor/vscode-jsonrpc/package.json")), + fs.readFileSync(path.join(root, "node_modules/vscode-jsonrpc/package.json")), + ); + const lspOutput = path.join(root, "tsc/internal/lsp/lsproto/lsp_generated.go"); + const timestamp = fs.statSync(lspOutput).mtimeMs; + const current = await x("npx", ["hereby", "generate:lsp"], { throwOnError: true, nodeOptions: { cwd: root } }); + assert.match(current.stdout, /LSP bindings are up to date/); + assert.doesNotMatch(current.stdout, /Using vscode-languageclient/); + assert.equal(fs.statSync(lspOutput).mtimeMs, timestamp); +}); + test("bundled generation skips unchanged library outputs", async () => { const root = path.resolve(import.meta.dirname, "../../.."); const generate = () => x("go", ["-C", "./tsc", "generate", "./internal/bundled"], { throwOnError: true, nodeOptions: { cwd: root } }); @@ -174,58 +271,42 @@ test("bundled generation tracks libraries, validates inputs, and supports force" outputs: ["libs_generated.go", "embed_generated.go"], commands: [["go", "run", path.join(root, "tsc/internal/bundled/generate.go")]], }; - const command = generatorArgs(options); - const generate = (...args: string[]) => x(process.execPath, generatorArgs(options, args), { throwOnError: true, nodeOptions: { cwd: root } }); + const generate = (force = false) => cache({ ...options, force }); await generate(); const files = ["libs_generated.go", "embed_generated.go"].map(file => path.join(directory, file)); const originals = files.map(file => fs.readFileSync(file)); - assert.match((await generate()).stdout, /Codegen outputs are up to date\./); + assert.equal(await generate(), true); const added = path.join(libs, "lib.second.d.ts"); fs.writeFileSync(added, content); - assert.match((await generate()).stdout, /Generated codegen outputs\./); + assert.equal(await generate(), false); for (const file of files) assert.match(fs.readFileSync(file, "utf8"), /lib\.second\.d\.ts/); fs.rmSync(added); - assert.match((await generate()).stdout, /Generated codegen outputs\./); + assert.equal(await generate(), false); for (const [index, file] of files.entries()) { assert.deepEqual(fs.readFileSync(file), originals[index]); fs.writeFileSync(file, "modified"); - assert.match((await generate()).stdout, /Generated codegen outputs\./); + assert.equal(await generate(), false); assert.deepEqual(fs.readFileSync(file), originals[index]); fs.rmSync(file, { recursive: true, maxRetries: 10 }); - assert.match((await generate()).stdout, /Generated codegen outputs\./); + assert.equal(await generate(), false); assert.deepEqual(fs.readFileSync(file), originals[index]); } fs.writeFileSync(library, content.replace("string", "number")); - assert.match((await generate()).stdout, /Generated codegen outputs\./); + assert.equal(await generate(), false); fs.writeFileSync(library, content.replaceAll("\n", "\r\n")); - await assert.rejects(async () => await generate(), error => { - assert.match((error as { output: { stderr: string; }; }).output.stderr, /must use LF line endings/); - return true; - }); + await assert.rejects(generate()); fs.writeFileSync(library, content); - assert.match((await generate()).stdout, /Generated codegen outputs\./); + assert.equal(await generate(), false); fs.mkdirSync(path.join(libs, "unexpected")); - await assert.rejects(async () => await generate(), error => { - assert.match((error as { output: { stderr: string; }; }).output.stderr, /unexpected entry/); - return true; - }); + await assert.rejects(generate()); fs.rmdirSync(path.join(libs, "unexpected")); - assert.match((await generate()).stdout, /Generated codegen outputs\./); + assert.equal(await generate(), false); fs.writeFileSync(notice, "Changed fixture notice\n\n"); - await assert.rejects(async () => await generate(), error => { - assert.match((error as { output: { stderr: string; }; }).output.stderr, /must start with/); - return true; - }); + await assert.rejects(generate()); fs.writeFileSync(notice, "Fixture notice\n\n"); - assert.match((await generate()).stdout, /Generated codegen outputs\./); - assert.match((await generate("--force")).stdout, /Generated codegen outputs\./); - assert.match((await generate()).stdout, /Codegen outputs are up to date\./); - const forced = await x(process.execPath, command, { - throwOnError: true, - nodeOptions: { cwd: root, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, - }); - assert.match(forced.stdout, /Generated codegen outputs\./); - assert.match((await generate()).stdout, /Codegen outputs are up to date\./); + assert.equal(await generate(), false); + assert.equal(await generate(true), false); + assert.equal(await generate(), true); }); test("stringer generation tracks declared inputs, options, output, and force", async context => { @@ -236,60 +317,46 @@ test("stringer generation tracks declared inputs, options, output, and force", a const extra = path.join(directory, "extra.go"); const output = path.join(directory, "kind_stringer_generated.go"); fs.writeFileSync(source, "package probe\n\ntype Kind int\n\nconst KindFirst Kind = 0\n"); - const command = (...args: string[]) => - generatorArgs({ + const generate = (trimPrefix = false, force = false) => + cache({ cwd: directory, inputs: ["kind.go"], outputs: ["kind_stringer_generated.go"], + env: { GOFILE: "kind.go" }, + force, commands: [ - ["go", "tool", "golang.org/x/tools/cmd/stringer", "-type=Kind", "-output=kind_stringer_generated.go", ...args.filter(arg => arg !== "--force")], + ["go", "tool", "golang.org/x/tools/cmd/stringer", "-type=Kind", "-output=kind_stringer_generated.go", ...(trimPrefix ? ["-trimprefix=Kind"] : [])], ["dprint", "fmt", "kind_stringer_generated.go"], ], - }, args.filter(arg => arg === "--force")); - const generate = (...args: string[]) => - x(process.execPath, command(...args), { - throwOnError: true, - nodeOptions: { cwd: directory, env: { ...process.env, GOFILE: "kind.go" } }, }); await generate(); const timestamp = fs.statSync(output).mtimeMs; - assert.match((await generate()).stdout, /Codegen outputs are up to date\./); + assert.equal(await generate(), true); assert.equal(fs.statSync(output).mtimeMs, timestamp); fs.writeFileSync(extra, "package probe\n\nconst Unrelated = 1\n"); - assert.match((await generate()).stdout, /Codegen outputs are up to date\./); + assert.equal(await generate(), true); fs.appendFileSync(source, "\nconst KindSecond Kind = 1\n"); - assert.match((await generate()).stdout, /Generated codegen outputs\./); + assert.equal(await generate(), false); assert.match(fs.readFileSync(output, "utf8"), /KindSecond/); fs.rmSync(extra); - assert.match((await generate()).stdout, /Codegen outputs are up to date\./); + assert.equal(await generate(), true); fs.writeFileSync(source, "package probe\n\ntype Kind int\n\nconst KindFirst Kind = 0\n"); - assert.match((await generate()).stdout, /Generated codegen outputs\./); + assert.equal(await generate(), false); assert.doesNotMatch(fs.readFileSync(output, "utf8"), /KindSecond/); - assert.match((await generate("-trimprefix=Kind")).stdout, /Generated codegen outputs\./); + assert.equal(await generate(true), false); assert.match(fs.readFileSync(output, "utf8"), /_Kind_name = "First"/); - assert.match((await generate("-trimprefix=Kind")).stdout, /Codegen outputs are up to date\./); + assert.equal(await generate(true), true); fs.appendFileSync(output, "\n"); - assert.match((await generate("-trimprefix=Kind")).stdout, /Generated codegen outputs\./); + assert.equal(await generate(true), false); fs.rmSync(output); - assert.match((await generate("-trimprefix=Kind")).stdout, /Generated codegen outputs\./); - assert.match((await generate("-trimprefix=Kind", "--force")).stdout, /Generated codegen outputs\./); - assert.match((await generate("-trimprefix=Kind")).stdout, /Codegen outputs are up to date\./); - const explicit = await x(process.execPath, command("-trimprefix=Kind"), { - throwOnError: true, - nodeOptions: { cwd: directory, env: { ...process.env, GOFILE: "" } }, - }); - assert.match(explicit.stdout, /Codegen outputs are up to date\./); - const forced = await x(process.execPath, command("-trimprefix=Kind"), { - throwOnError: true, - nodeOptions: { cwd: directory, env: { ...process.env, GOFILE: "kind.go", TSGO_HEREBY_FORCE: "1" } }, - }); - assert.match(forced.stdout, /Generated codegen outputs\./); - assert.match((await generate("-trimprefix=Kind")).stdout, /Codegen outputs are up to date\./); + assert.equal(await generate(true), false); + assert.equal(await generate(true, true), false); + assert.equal(await generate(true), true); fs.appendFileSync(source, "\nconst KindInvalid Kind = missing\n"); - await assert.rejects(async () => await generate("-trimprefix=Kind")); + await assert.rejects(generate(true)); fs.writeFileSync(source, "package probe\n\ntype Kind int\n\nconst KindFirst Kind = 0\n"); - assert.match((await generate("-trimprefix=Kind")).stdout, /Generated codegen outputs\./); - assert.match((await generate("-trimprefix=Kind")).stdout, /Codegen outputs are up to date\./); + assert.equal(await generate(true), false); + assert.equal(await generate(true), true); }); test("moq generation tracks interface inputs, output, and force", async context => { @@ -311,28 +378,18 @@ test("moq generation tracks interface inputs, output, and force", async context ["dprint", "fmt", "mock/mock_generated.go"], ], }; - const generate = (...args: string[]) => - x(process.execPath, generatorArgs(options, args), { - throwOnError: true, - nodeOptions: { cwd: directory }, - }); + const generate = (force = false) => cache({ ...options, force }); await generate(); const timestamp = fs.statSync(output).mtimeMs; - assert.match((await generate()).stdout, /Codegen outputs are up to date\./); + assert.equal(await generate(), true); assert.equal(fs.statSync(output).mtimeMs, timestamp); fs.writeFileSync(base, "package probe\n\ntype Base interface { Read() string; Close() error }\n"); - assert.match((await generate()).stdout, /Generated codegen outputs\./); + assert.equal(await generate(), false); assert.match(fs.readFileSync(output, "utf8"), /CloseFunc/); fs.rmSync(output); - assert.match((await generate()).stdout, /Generated codegen outputs\./); - assert.match((await generate("--force")).stdout, /Generated codegen outputs\./); - assert.match((await generate()).stdout, /Codegen outputs are up to date\./); - const forced = await x(process.execPath, generatorArgs(options), { - throwOnError: true, - nodeOptions: { cwd: directory, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, - }); - assert.match(forced.stdout, /Generated codegen outputs\./); - assert.match((await generate()).stdout, /Codegen outputs are up to date\./); + assert.equal(await generate(), false); + assert.equal(await generate(true), false); + assert.equal(await generate(), true); }); test("diagnostic generation tracks Go and locale outputs and supports force", async context => { @@ -351,42 +408,32 @@ test("diagnostic generation tracks Go and locale outputs and supports force", as ["dprint", "fmt", output, localized], ], }; - const generate = (...args: string[]) => - x(process.execPath, generatorArgs(options, args), { - throwOnError: true, - nodeOptions: { cwd: root }, - }); + const generate = (force = false) => cache({ ...options, force }); await generate(); const locales = fs.globSync("loc/*.json.gz", { cwd: directory }).map(file => path.join(directory, file)); assert.ok(locales.length > 0); const files = [output, localized, ...locales]; const timestamps = files.map(file => fs.statSync(file).mtimeMs); - assert.match((await generate()).stdout, /Codegen outputs are up to date\./); + assert.equal(await generate(), true); assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); const archive = fs.readFileSync(locales[0]); fs.rmSync(locales[0]); - assert.match((await generate()).stdout, /Generated codegen outputs\./); + assert.equal(await generate(), false); assert.deepEqual(fs.readFileSync(locales[0]), archive); fs.writeFileSync(locales[0], "modified"); - assert.match((await generate()).stdout, /Generated codegen outputs\./); + assert.equal(await generate(), false); assert.deepEqual(fs.readFileSync(locales[0]), archive); const unexpected = path.join(directory, "loc/unexpected.json.gz"); fs.writeFileSync(unexpected, "unexpected"); - assert.match((await generate()).stdout, /Generated codegen outputs\./); + assert.equal(await generate(), false); assert.equal(fs.existsSync(unexpected), false); fs.rmSync(path.join(directory, "loc"), { recursive: true }); - assert.match((await generate()).stdout, /Generated codegen outputs\./); + assert.equal(await generate(), false); assert.deepEqual(fs.readFileSync(locales[0]), archive); fs.rmSync(localized); - assert.match((await generate()).stdout, /Generated codegen outputs\./); - assert.match((await generate("--force")).stdout, /Generated codegen outputs\./); - assert.match((await generate()).stdout, /Codegen outputs are up to date\./); - const forced = await x(process.execPath, generatorArgs(options), { - throwOnError: true, - nodeOptions: { cwd: root, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, - }); - assert.match(forced.stdout, /Generated codegen outputs\./); - assert.match((await generate()).stdout, /Codegen outputs are up to date\./); + assert.equal(await generate(), false); + assert.equal(await generate(true), false); + assert.equal(await generate(), true); }); test("Unicode generation skips unchanged tables and formatting", async () => { @@ -464,7 +511,7 @@ test("enum generation skips unchanged outputs and Go verification", async () => assert.match((await generate()).stdout, /Enums are up to date\./); }); -test("AST generation forwards force to all schema generators", async () => { +test("AST generation forwards force to schema generators and the kind stringer", async () => { const root = path.resolve(import.meta.dirname, "../../.."); const generate = (force = false) => x("npx", ["hereby", "generate:ast", ...(force ? ["--force"] : [])], { throwOnError: true, nodeOptions: { cwd: root } }); await generate(); @@ -472,8 +519,14 @@ test("AST generation forwards force to all schema generators", async () => { assert.match(forced.stdout, /Wrote .*encoder_generated\.go/); assert.match(forced.stdout, /Wrote .*ast_generated\.go/); assert.match(forced.stdout, /Generated .*ast\.generated\.ts/); + assert.match(forced.stdout, /Generated codegen outputs\./); + const output = path.join(root, "tsc/internal/ast/kind_stringer_generated.go"); + const timestamp = fs.statSync(output).mtimeMs; assert.doesNotMatch(forced.stdout, /\$ node .*tools\/scripts\/tsc\/generate\.ts/); - assert.doesNotMatch((await generate()).stdout, /(?:Wrote|Generated) /); + const current = await generate(); + assert.doesNotMatch(current.stdout, /(?:Wrote|Generated) /); + assert.match(current.stdout, /Codegen outputs are up to date\./); + assert.equal(fs.statSync(output).mtimeMs, timestamp); }); test("package generation forwards force to AST, encoder, and sync generators", async () => { diff --git a/tools/scripts/tsc/generate-go-ast.ts b/tools/scripts/tsc/generate-go-ast.ts index 7a85954a9836c..2f1bf452d09af 100644 --- a/tools/scripts/tsc/generate-go-ast.ts +++ b/tools/scripts/tsc/generate-go-ast.ts @@ -986,7 +986,7 @@ function generateKind(): string { w.write(""); w.write("package ast"); w.write(""); - w.write("//go:generate npm run --silent cache -- --input $GOFILE --output kind_stringer_generated.go --command go tool golang.org/x/tools/cmd/stringer -type=Kind -output=kind_stringer_generated.go --command dprint fmt kind_stringer_generated.go"); + w.write("//go:generate npx hereby generate:ast-stringer"); w.write(""); w.write("type Kind int16"); w.write(""); diff --git a/tsc/internal/ast/kind_generated.go b/tsc/internal/ast/kind_generated.go index 46cf07a1d2dd1..c74eb65e60c18 100644 --- a/tsc/internal/ast/kind_generated.go +++ b/tsc/internal/ast/kind_generated.go @@ -2,7 +2,7 @@ package ast -//go:generate npm run --silent cache -- --input $GOFILE --output kind_stringer_generated.go --command go tool golang.org/x/tools/cmd/stringer -type=Kind -output=kind_stringer_generated.go --command dprint fmt kind_stringer_generated.go +//go:generate npx hereby generate:ast-stringer type Kind int16 diff --git a/tsc/internal/bundled/bundled.go b/tsc/internal/bundled/bundled.go index c2f7d59c04f9a..c1be755cb6833 100644 --- a/tsc/internal/bundled/bundled.go +++ b/tsc/internal/bundled/bundled.go @@ -11,7 +11,7 @@ import ( "github.com/microsoft/TypeScript/tsc/internal/vfs" ) -//go:generate npm run --silent cache -- --input generate.go --input CopyrightNotice.txt --input "libs/*" --output libs_generated.go --output embed_generated.go --command go run generate.go +//go:generate npx hereby generate:bundled // Define the below here to consolidate documentation. diff --git a/tsc/internal/checker/types.go b/tsc/internal/checker/types.go index 56891bb8804b7..2321366d29f38 100644 --- a/tsc/internal/checker/types.go +++ b/tsc/internal/checker/types.go @@ -11,7 +11,7 @@ import ( "github.com/microsoft/TypeScript/tsc/internal/evaluator" ) -//go:generate npm run --silent cache -- --input $GOFILE --output stringer_generated.go --command go tool golang.org/x/tools/cmd/stringer -type=SignatureKind -output=stringer_generated.go --command dprint fmt stringer_generated.go +//go:generate npx hereby generate:checker // ParseFlags diff --git a/tsc/internal/core/compileroptions.go b/tsc/internal/core/compileroptions.go index 106343c2678b9..21457c1e25f3b 100644 --- a/tsc/internal/core/compileroptions.go +++ b/tsc/internal/core/compileroptions.go @@ -9,8 +9,7 @@ import ( "github.com/microsoft/TypeScript/tsc/internal/tspath" ) -//go:generate npm run --silent cache -- --input $GOFILE --output modulekind_stringer_generated.go --command go tool golang.org/x/tools/cmd/stringer -type=ModuleKind -trimprefix=ModuleKind -output=modulekind_stringer_generated.go --command dprint fmt modulekind_stringer_generated.go -//go:generate npm run --silent cache -- --input $GOFILE --output scripttarget_stringer_generated.go --command go tool golang.org/x/tools/cmd/stringer -type=ScriptTarget -trimprefix=ScriptTarget -output=scripttarget_stringer_generated.go --command dprint fmt scripttarget_stringer_generated.go +//go:generate npx hereby generate:compileroptions // CompilerOptions contains the compiler options exposed by the API. type CompilerOptions struct { diff --git a/tsc/internal/core/languagevariant.go b/tsc/internal/core/languagevariant.go index 83c3141aefd36..2a98eec3f78fd 100644 --- a/tsc/internal/core/languagevariant.go +++ b/tsc/internal/core/languagevariant.go @@ -1,6 +1,6 @@ package core -//go:generate npm run --silent cache -- --input $GOFILE --output languagevariant_stringer_generated.go --command go tool golang.org/x/tools/cmd/stringer -type=LanguageVariant -output=languagevariant_stringer_generated.go --command dprint fmt languagevariant_stringer_generated.go +//go:generate npx hereby generate:languagevariant type LanguageVariant int32 diff --git a/tsc/internal/core/scriptkind.go b/tsc/internal/core/scriptkind.go index 1813d9b5fe70d..78699915f6469 100644 --- a/tsc/internal/core/scriptkind.go +++ b/tsc/internal/core/scriptkind.go @@ -1,6 +1,6 @@ package core -//go:generate npm run --silent cache -- --input $GOFILE --output scriptkind_stringer_generated.go --command go tool golang.org/x/tools/cmd/stringer -type=ScriptKind -output=scriptkind_stringer_generated.go --command dprint fmt scriptkind_stringer_generated.go +//go:generate npx hereby generate:scriptkind type ScriptKind int32 diff --git a/tsc/internal/core/tristate.go b/tsc/internal/core/tristate.go index 4c6b4032ae98b..61401cf51976d 100644 --- a/tsc/internal/core/tristate.go +++ b/tsc/internal/core/tristate.go @@ -1,6 +1,6 @@ package core -//go:generate npm run --silent cache -- --input $GOFILE --output tristate_stringer_generated.go --command go tool golang.org/x/tools/cmd/stringer -type=Tristate -output=tristate_stringer_generated.go --command dprint fmt tristate_stringer_generated.go +//go:generate npx hereby generate:tristate // Tristate diff --git a/tsc/internal/diagnostics/diagnostics.go b/tsc/internal/diagnostics/diagnostics.go index 2b0e01de4f76f..8c84a7a72a5dd 100644 --- a/tsc/internal/diagnostics/diagnostics.go +++ b/tsc/internal/diagnostics/diagnostics.go @@ -13,8 +13,7 @@ import ( "golang.org/x/text/language" ) -//go:generate npm run --silent cache -- --input generate.go --input diagnosticMessages.json --input extraDiagnosticMessages.json --input "../{collections,json}/*.go" --input "../locale/lcl/*/diagnosticMessages/diagnosticMessages.generated.json.lcl" --exclude "**/*_test.go" --output diagnostics_generated.go --output loc_generated.go --output "loc/*.json.gz" --command go run generate.go -diagnostics diagnostics_generated.go -loc loc_generated.go -locdir loc --command dprint fmt diagnostics_generated.go loc_generated.go -//go:generate npm run --silent cache -- --input $GOFILE --output stringer_generated.go --command go tool golang.org/x/tools/cmd/stringer -type=Category -output=stringer_generated.go --command dprint fmt stringer_generated.go +//go:generate npx hereby generate:diagnostics type Category int32 diff --git a/tsc/internal/ls/autoimport/export.go b/tsc/internal/ls/autoimport/export.go index d592ac2528977..8d43bb464bca7 100644 --- a/tsc/internal/ls/autoimport/export.go +++ b/tsc/internal/ls/autoimport/export.go @@ -8,7 +8,7 @@ import ( "github.com/microsoft/TypeScript/tsc/internal/tspath" ) -//go:generate npm run --silent cache -- --input $GOFILE --output export_stringer_generated.go --command go tool golang.org/x/tools/cmd/stringer -type=ExportSyntax -output=export_stringer_generated.go --command dprint fmt export_stringer_generated.go +//go:generate npx hereby generate:autoimport // ModuleID uniquely identifies a module across multiple declarations. // If the export is from an ambient module declaration, this is the module name. diff --git a/tsc/internal/lsp/lsproto/_generate/fetchModel.mts b/tsc/internal/lsp/lsproto/_generate/fetchModel.mts index ee3a9111d1441..02db3453f5de6 100755 --- a/tsc/internal/lsp/lsproto/_generate/fetchModel.mts +++ b/tsc/internal/lsp/lsproto/_generate/fetchModel.mts @@ -12,28 +12,35 @@ const __dirname = path.dirname(__filename); const metaModelPath = path.join(__dirname, "metaModel.json"); const metaModelSchemaPath = path.join(__dirname, "metaModelSchema.mts"); -// Resolve the vscode-languageclient version from the root package-lock.json. -const lockfilePath = path.resolve(__dirname, "../../../../../package-lock.json"); -const lockfile = JSON.parse(fs.readFileSync(lockfilePath, "utf-8")); -const clientVersion: string = lockfile.packages["node_modules/vscode-languageclient"].version; - -const ref = `release/client/${clientVersion}`; -console.log(`Using vscode-languageclient@${clientVersion}`); - -const metaModelURL = `https://raw.githubusercontent.com/microsoft/vscode-languageserver-node/${ref}/protocol/metaModel.json`; -const metaModelSchemaURL = `https://raw.githubusercontent.com/microsoft/vscode-languageserver-node/${ref}/tools/src/metaModel.ts`; - -const metaModelResponse = await fetch(metaModelURL); -const metaModel = await metaModelResponse.text(); -fs.writeFileSync(metaModelPath, metaModel); - -const metaModelSchemaResponse = await fetch(metaModelSchemaURL); -let metaModelSchema = await metaModelSchemaResponse.text(); - -// Patch the schema to add omitzeroValue property to Property type -metaModelSchema = metaModelSchema.replace( - /(\t \* Whether the property is deprecated or not\. If deprecated\n\t \* the property contains the deprecation message\.\n\t \*\/\n\tdeprecated\?: string;)\n}/m, - `$1\n\n\t/**\n\t * Whether this property uses omitzero without being a pointer.\n\t * Custom extension for special value types.\n\t */\n\tomitzeroValue?: boolean;\n}`, -); - -fs.writeFileSync(metaModelSchemaPath, metaModelSchema); +export default async function fetchModel() { + const lockfilePath = path.resolve(__dirname, "../../../../../package-lock.json"); + const lockfile = JSON.parse(fs.readFileSync(lockfilePath, "utf-8")); + const clientVersion: string = lockfile.packages["node_modules/vscode-languageclient"].version; + + const ref = `release/client/${clientVersion}`; + console.log(`Using vscode-languageclient@${clientVersion}`); + + const metaModelURL = `https://raw.githubusercontent.com/microsoft/vscode-languageserver-node/${ref}/protocol/metaModel.json`; + const metaModelSchemaURL = `https://raw.githubusercontent.com/microsoft/vscode-languageserver-node/${ref}/tools/src/metaModel.ts`; + + const metaModelResponse = await fetch(metaModelURL); + if (!metaModelResponse.ok) throw new Error(`Failed to fetch ${metaModelURL}: ${metaModelResponse.status}`); + const metaModel = await metaModelResponse.text(); + JSON.parse(metaModel); + + const metaModelSchemaResponse = await fetch(metaModelSchemaURL); + if (!metaModelSchemaResponse.ok) throw new Error(`Failed to fetch ${metaModelSchemaURL}: ${metaModelSchemaResponse.status}`); + let metaModelSchema = await metaModelSchemaResponse.text(); + + metaModelSchema = metaModelSchema.replace( + /(\t \* Whether the property is deprecated or not\. If deprecated\n\t \* the property contains the deprecation message\.\n\t \*\/\n\tdeprecated\?: string;)\n}/m, + `$1\n\n\t/**\n\t * Whether this property uses omitzero without being a pointer.\n\t * Custom extension for special value types.\n\t */\n\tomitzeroValue?: boolean;\n}`, + ); + + fs.writeFileSync(metaModelPath, metaModel); + fs.writeFileSync(metaModelSchemaPath, metaModelSchema); +} + +if (process.argv[1] === __filename) { + await fetchModel(); +} diff --git a/tsc/internal/lsp/lsproto/_generate/generate.mts b/tsc/internal/lsp/lsproto/_generate/generate.mts index 6f65a9af99c1e..68a141609a1b0 100755 --- a/tsc/internal/lsp/lsproto/_generate/generate.mts +++ b/tsc/internal/lsp/lsproto/_generate/generate.mts @@ -3603,7 +3603,7 @@ function getLocationUriProperty(structure: Structure) { /** * Main function */ -async function main() { +export default async function generate() { collectTypeDefinitions(); const generatedCode = generateCode(); fs.writeFileSync(out, generatedCode); @@ -3616,7 +3616,6 @@ async function main() { console.log(`Successfully generated ${out}`); } -main().catch(e => { - console.error(e); - process.exit(1); -}); +if (process.argv[1] === __filename) { + await generate(); +} diff --git a/tsc/internal/project/project.go b/tsc/internal/project/project.go index 6e438ca1eba5a..cb9883c152068 100644 --- a/tsc/internal/project/project.go +++ b/tsc/internal/project/project.go @@ -40,7 +40,7 @@ func SyntheticProgramID(path tspath.Path) (int, bool) { return id, err == nil && id > 0 } -//go:generate npm run --silent cache -- --input $GOFILE --output project_stringer_generated.go --command go tool golang.org/x/tools/cmd/stringer -type=Kind -trimprefix=Kind -output=project_stringer_generated.go --command dprint fmt project_stringer_generated.go +//go:generate npx hereby generate:project type Kind int diff --git a/tsc/internal/stringutil/generate.go b/tsc/internal/stringutil/generate.go index 408eb756548cf..be4a3c029e7bb 100644 --- a/tsc/internal/stringutil/generate.go +++ b/tsc/internal/stringutil/generate.go @@ -1,3 +1,3 @@ package stringutil -//go:generate node ./_scripts/generate-unicode-data.mts +//go:generate npx hereby generate:unicode diff --git a/tsc/internal/testutil/projecttestutil/projecttestutil.go b/tsc/internal/testutil/projecttestutil/projecttestutil.go index 396ec6501f43a..7375a7d4fc79f 100644 --- a/tsc/internal/testutil/projecttestutil/projecttestutil.go +++ b/tsc/internal/testutil/projecttestutil/projecttestutil.go @@ -23,9 +23,7 @@ import ( "github.com/microsoft/TypeScript/tsc/internal/vfs/vfstest" ) -//go:generate npm run --silent cache -- --input $GOFILE --input ../../project/client.go --output clientmock_generated.go --command go tool github.com/matryer/moq -stub -fmt goimports -pkg projecttestutil -out clientmock_generated.go ../../project Client --command dprint fmt clientmock_generated.go - -//go:generate npm run --silent cache -- --input $GOFILE --input ../../project/ata/ata.go --output npmexecutormock_generated.go --command go tool github.com/matryer/moq -stub -fmt goimports -pkg projecttestutil -out npmexecutormock_generated.go ../../project/ata NpmExecutor --command dprint fmt npmexecutormock_generated.go +//go:generate npx hereby generate:projecttestutil const ( TestTypingsLocation = "/home/src/Library/Caches/typescript" diff --git a/tsc/internal/vfs/vfs.go b/tsc/internal/vfs/vfs.go index f32f96e5aefcf..0e858e53de9c7 100644 --- a/tsc/internal/vfs/vfs.go +++ b/tsc/internal/vfs/vfs.go @@ -5,7 +5,7 @@ import ( "time" ) -//go:generate npm run --silent cache -- --input $GOFILE --output vfsmock/mock_generated.go --command go tool github.com/matryer/moq -fmt goimports -out vfsmock/mock_generated.go -pkg vfsmock . FS --command dprint fmt vfsmock/mock_generated.go +//go:generate npx hereby generate:vfs // FS is a file system abstraction. type FS interface { diff --git a/tsc/internal/vfs/vfsmatch/vfsmatch.go b/tsc/internal/vfs/vfsmatch/vfsmatch.go index cf118a466249a..4f4acf7454cc0 100644 --- a/tsc/internal/vfs/vfsmatch/vfsmatch.go +++ b/tsc/internal/vfs/vfsmatch/vfsmatch.go @@ -12,7 +12,7 @@ import ( "github.com/microsoft/TypeScript/tsc/internal/vfs" ) -//go:generate npm run --silent cache -- --input $GOFILE --output stringer_generated.go --command go tool golang.org/x/tools/cmd/stringer -type=Usage -trimprefix=Usage -output=stringer_generated.go --command dprint fmt stringer_generated.go +//go:generate npx hereby generate:vfsmatch // This file implements the glob matching algorithm specified in MATCHING_ALGORITHM.md. From 193f7e3b90fbe6531f31d17212439ba30a9ad15a Mon Sep 17 00:00:00 2001 From: Wesley Wigham Date: Mon, 21 Sep 2026 12:56:33 -0700 Subject: [PATCH 5/6] Rename tasks --- .../workflows/accept-baselines-fix-lints.yaml | 2 +- .github/workflows/ci.yml | 2 +- CONTRIBUTING.md | 9 ++++---- Herebyfile.mjs | 22 +++++++++---------- tools/scripts/gen/generatedFile.test.mts | 22 +++++++++---------- 5 files changed, 29 insertions(+), 28 deletions(-) diff --git a/.github/workflows/accept-baselines-fix-lints.yaml b/.github/workflows/accept-baselines-fix-lints.yaml index 5d46db691a62c..83bcca9b24123 100644 --- a/.github/workflows/accept-baselines-fix-lints.yaml +++ b/.github/workflows/accept-baselines-fix-lints.yaml @@ -30,7 +30,7 @@ jobs: git config user.email "290192711+typescript-automation[bot]@users.noreply.github.com" git config user.name "typescript-automation[bot]" npm ci - npx hereby generate + npx hereby generate:go npx hereby test || true npx hereby baseline-accept npx hereby lint --fix diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 009dac1aead28..6269414fd6d19 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -293,7 +293,7 @@ jobs: node-version: '>=22.18.0' - uses: ./.github/actions/setup-go - run: npm ci - - run: npx hereby generate + - run: npx hereby generate:go - run: npx hereby generate:enums - run: npx hereby generate:ast - run: npx hereby generate:vendor diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8d44df486ca45..462f0e44e9fef 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -103,8 +103,8 @@ npx hereby build # Build the native compiler into built/local/tsc npx hereby test # Run compiler and language-service Go tests npx hereby test:all # Also run benchmarks, tools, and API tests npx hereby lint # Run custom golangci-lint for both Go modules -npx hereby generate # Regenerate compiler sources and bundled assets -npx hereby generate:all # Also regenerate AST, LSP, APIs, localization, and vendored files +npx hereby generate:go # Run the generators also exposed through go generate +npx hereby generate # Regenerate all sources, assets, localization, and vendored files npx hereby format # Format Go, TypeScript, JSON, and YAML npx hereby check:format # Check formatting without changing files npx hereby tidy # Tidy both modules and synchronize go.work @@ -113,8 +113,9 @@ npx hereby tidy # Tidy both modules and synchronize go.work Generation is owned by `Herebyfile.mjs`; Go directives forward to the same `generate:*` tasks for compatibility. Use a subtask such as `npx hereby generate:diagnostics` to run one generator group, and pass `--force` -to bypass incremental caches. `generate:all` fetches the pinned LSP model when -its local cache is missing or stale. +to bypass incremental caches. `generate` includes every generator and fetches +the pinned LSP model when its local cache is missing or stale. Use +`generate:go` for the narrower scope of `go generate ./...` in the `tsc` module. Package-specific commands: diff --git a/Herebyfile.mjs b/Herebyfile.mjs index 18ff7f0289c39..76d2794f3049f 100644 --- a/Herebyfile.mjs +++ b/Herebyfile.mjs @@ -372,16 +372,16 @@ export const cleanBuilt = task({ /** @type {(() => Promise)[]} */ const goGenerateActions = []; -async function runGenerate() { +async function runGenerateGo() { for (const generate of goGenerateActions) { await generate(); } } -export const generate = task({ - name: "generate", +export const generateGo = task({ + name: "generate:go", description: "Runs the project's Go code generators directly. Pass --force to regenerate unchanged files.", - run: runGenerate, + run: runGenerateGo, }); const getGoGenerateEnvironment = memoize(async () => { @@ -1292,20 +1292,20 @@ export const generateVendor = task({ run: runGenerateVendor, }); -const generateAllCompiler = task({ - name: "generate:all:compiler", +const generateCompiler = task({ + name: "generate:compiler", hiddenFromTaskList: true, dependencies: [generateAST, generateLSP], run: async () => { - await runGenerate(); + await runGenerateGo(); await runGenerateEnums(); }, }); -export const generateAll = task({ - name: "generate:all", +export const generate = task({ + name: "generate", description: "Runs all code generation, including AST, LSP, APIs, extension localization, and vendored dependencies.", - dependencies: [generateAllCompiler, generateSync, generateExtensionTest, generateVendor], + dependencies: [generateCompiler, generateSync, generateExtensionTest, generateVendor], }); const coverageDir = path.join(__dirname, "coverage"); @@ -1718,7 +1718,7 @@ async function runFormat() { export const validate = task({ name: "validate", description: "Generates, builds, tests, lints, and formats the repo. Pass --api to include API tests, or --all to include all code generation and ancillary repository tests Benchmarks are separate: test:benchmarks and test:benchmarks:api.", - dependencies: [options.all ? generateAll : generate], + dependencies: [options.all ? generate : generateGo], run: async () => { await generateLibs(builtLocal); await buildTsc({ extraFlags: options.release ? getReleaseBuildFlags() : [] }); diff --git a/tools/scripts/gen/generatedFile.test.mts b/tools/scripts/gen/generatedFile.test.mts index 6eb0c0f499c6d..38e9d53872fb4 100644 --- a/tools/scripts/gen/generatedFile.test.mts +++ b/tools/scripts/gen/generatedFile.test.mts @@ -11,7 +11,7 @@ import ts from "typescript"; import cache from "./cache.mts"; import { GeneratedFile } from "./generatedFile.mts"; -test("validate generates before building and reuses generate:all", async () => { +test("validate generates before building and selects the generation scope", async () => { const fileName = path.resolve(import.meta.dirname, "../../../Herebyfile.mjs"); const source = ts.createSourceFile(fileName, fs.readFileSync(fileName, "utf8"), ts.ScriptTarget.Latest, true, ts.ScriptKind.JS); const declaration = source.statements.filter(ts.isVariableStatement) @@ -24,19 +24,19 @@ test("validate generates before building and reuses generate:all", async () => { calls.push(name); }; const generate = { run: action("generate") }; - const generateAll = { run: action("generate:all") }; + const generateGo = { run: action("generate:go") }; const build = { run: action("build") }; const validation = runInNewContext(declaration.initializer.getText(source), { task: (spec: unknown) => spec, options, generate, - generateAll, + generateGo, build, builtLocal: "./built/local", generateLibs: action("lib"), buildTsc: action("build"), getReleaseBuildFlags: () => [], - runGenerate: action("generate"), + runGenerateGo: action("generate:go"), runGenerateEnums: action("generate:enums"), runGenerateAPI: action("generate:api"), runGenerateExtension: action("generate:extension"), @@ -56,13 +56,13 @@ test("validate generates before building and reuses generate:all", async () => { }) as { dependencies: { run: () => Promise; }[]; run: () => Promise; }; await Promise.all(validation.dependencies.map(dependency => dependency.run())); await validation.run(); - assert.equal(calls[0], "all" in options ? "generate:all" : "generate"); + assert.equal(calls[0], "all" in options ? "generate" : "generate:go"); assert.ok(calls.indexOf("build") > 0); assert.ok(calls.indexOf("build") < calls.indexOf("test:tsc")); assert.equal(calls.includes("test:api"), "api" in options || "all" in options); assert.equal(calls.includes("test:tools"), "all" in options); if ("all" in options) { - assert.deepEqual(calls.filter(name => name.startsWith("generate")), ["generate:all"]); + assert.deepEqual(calls.filter(name => name.startsWith("generate")), ["generate"]); } } }); @@ -193,9 +193,9 @@ test("generated files track each output independently", context => { assert.equal(files[1].isCurrent(), false); }); -test("Hereby generation runs Go generators directly and shares caches with Go fallback", async () => { +test("generate:go runs Go generators directly and shares caches with Go fallback", async () => { const root = path.resolve(import.meta.dirname, "../../.."); - const generate = () => x("npx", ["hereby", "generate"], { throwOnError: true, nodeOptions: { cwd: root } }); + const generate = () => x("npx", ["hereby", "generate:go"], { throwOnError: true, nodeOptions: { cwd: root } }); const first = await generate(); assert.doesNotMatch(first.stdout, /\$ go generate|npm run --silent cache|\$ node .*generate-unicode-data/); const files = fs.globSync(["tsc/internal/**/*generated.go", "packages/typescript/src/api/proto.generated.ts"], { cwd: root }); @@ -210,16 +210,16 @@ test("Hereby generation runs Go generators directly and shares caches with Go fa assert.equal(nested.stdout.match(/Codegen outputs are up to date/g)?.length, 2); }); -test("generate:all includes standalone generators without Go traversal", async () => { +test("generate includes standalone generators without Go traversal", async () => { const root = path.resolve(import.meta.dirname, "../../.."); - const { stdout } = await x("npx", ["hereby", "generate:all"], { throwOnError: true, nodeOptions: { cwd: root } }); + const { stdout } = await x("npx", ["hereby", "generate"], { throwOnError: true, nodeOptions: { cwd: root } }); const log = stripVTControlCharacters(stdout); const eventIndex = (event: string) => { const index = log.indexOf(event); assert.ok(index >= 0, `Missing task event: ${event}`); return index; }; - const compilerStart = eventIndex("Starting generate:all:compiler"); + const compilerStart = eventIndex("Starting generate:compiler"); assert.ok(eventIndex("Finished generate:ast ") < compilerStart); assert.ok(eventIndex("Finished generate:lsp ") < compilerStart); assert.ok(eventIndex("Starting generate:sync") < compilerStart); From 699651ee4a915fbf20bbe6fc5c14038c0f150e26 Mon Sep 17 00:00:00 2001 From: Wesley Wigham Date: Mon, 21 Sep 2026 14:02:48 -0700 Subject: [PATCH 6/6] PR feedback --- .github/workflows/accept-baselines-fix-lints.yaml | 2 +- .github/workflows/ci.yml | 8 +------- tools/scripts/gen/cache.mts | 8 +++++--- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/.github/workflows/accept-baselines-fix-lints.yaml b/.github/workflows/accept-baselines-fix-lints.yaml index 83bcca9b24123..5d46db691a62c 100644 --- a/.github/workflows/accept-baselines-fix-lints.yaml +++ b/.github/workflows/accept-baselines-fix-lints.yaml @@ -30,7 +30,7 @@ jobs: git config user.email "290192711+typescript-automation[bot]@users.noreply.github.com" git config user.name "typescript-automation[bot]" npm ci - npx hereby generate:go + npx hereby generate npx hereby test || true npx hereby baseline-accept npx hereby lint --fix diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6269414fd6d19..f9031e87522f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -293,13 +293,7 @@ jobs: node-version: '>=22.18.0' - uses: ./.github/actions/setup-go - run: npm ci - - run: npx hereby generate:go - - run: npx hereby generate:enums - - run: npx hereby generate:ast - - run: npx hereby generate:vendor - - run: node ./tsc/internal/lsp/lsproto/_generate/fetchModel.mts - - run: node ./tsc/internal/lsp/lsproto/_generate/generate.mts - - run: npx hereby generate:extension + - run: npx hereby generate - run: npx hereby check:scripts - run: git add . - run: git diff --staged --exit-code --stat diff --git a/tools/scripts/gen/cache.mts b/tools/scripts/gen/cache.mts index f309af3b0c8c5..0f5fb3cb1a329 100644 --- a/tools/scripts/gen/cache.mts +++ b/tools/scripts/gen/cache.mts @@ -83,8 +83,9 @@ export default async function cache({ ); const previous = artifacts(outputFiles()); const complete = () => outputs.every(pattern => expand([pattern], cwd).some(file => fs.statSync(file).isFile())); + const commandText = commands.map(command => command.join(" ")).join("; "); if (complete() && previous.every(file => file.isCurrent(force))) { - console.log("Codegen outputs are up to date."); + console.log(`skipped ${commandText}: codegen outputs are already up to date`); return true; } @@ -93,9 +94,10 @@ export default async function cache({ await x(command, args, { throwOnError: true, nodeOptions: { cwd, env: environment, stdio: "inherit" } }); } if (!complete()) throw new Error(`Generation did not produce all declared outputs: ${outputs.join(", ")}`); + const generatedFiles = outputFiles(); if (snapshotInputs().hash === before.hash) { - for (const file of artifacts(outputFiles())) file.markCurrent(); + for (const file of artifacts(generatedFiles)) file.markCurrent(); } - console.log("Generated codegen outputs."); + console.log(`exec ${commandText}:\n Outputs:\n ${generatedFiles.join("\n ")}`); return false; }