diff --git a/CHANGELOG.md b/CHANGELOG.md index 508ed0ce5c..0a5dafcc4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ See the [releases page](https://github.com/github/codeql-action/releases) for th ## [UNRELEASED] -No user facing changes. +- The `tools` input to the `init` and `setup-codeql` actions now accepts GitHub release URLs. The Action selects a compatible bundle from the specified release, including a per-language bundle when eligible. [#4179](https://github.com/github/codeql-action/pull/4179) ## 4.38.2 - 24 Sept 2026 diff --git a/init/action.yml b/init/action.yml index 7787a0a071..e2a0419b65 100644 --- a/init/action.yml +++ b/init/action.yml @@ -10,6 +10,8 @@ inputs: - A local path to a CodeQL Bundle tarball, or - The URL of a CodeQL Bundle tarball GitHub release asset, or + - A release URL from GitHub.com or the current GitHub instance, such as + https://github.com/github/codeql-action/releases/tag/codeql-bundle-v2.27.0. - A special value `linked` which uses the version of the CodeQL tools that the Action has been bundled with. - A special value `nightly` which uses the latest nightly version of the diff --git a/lib/entry-points.js b/lib/entry-points.js index f78a951f06..cc20e39e21 100644 --- a/lib/entry-points.js +++ b/lib/entry-points.js @@ -26953,7 +26953,7 @@ var require_parse2 = __commonJS({ "node_modules/semver/functions/parse.js"(exports2, module2) { "use strict"; var SemVer = require_semver(); - var parse3 = (version, options, throwErrors = false) => { + var parse4 = (version, options, throwErrors = false) => { if (version instanceof SemVer) { return version; } @@ -26966,7 +26966,7 @@ var require_parse2 = __commonJS({ throw er; } }; - module2.exports = parse3; + module2.exports = parse4; } }); @@ -26974,9 +26974,9 @@ var require_parse2 = __commonJS({ var require_valid = __commonJS({ "node_modules/semver/functions/valid.js"(exports2, module2) { "use strict"; - var parse3 = require_parse2(); + var parse4 = require_parse2(); var valid4 = (version, options) => { - const v = parse3(version, options); + const v = parse4(version, options); return v ? v.version : null; }; module2.exports = valid4; @@ -26987,9 +26987,9 @@ var require_valid = __commonJS({ var require_clean = __commonJS({ "node_modules/semver/functions/clean.js"(exports2, module2) { "use strict"; - var parse3 = require_parse2(); + var parse4 = require_parse2(); var clean3 = (version, options) => { - const s = parse3(version.trim().replace(/^[=v]+/, ""), options); + const s = parse4(version.trim().replace(/^[=v]+/, ""), options); return s ? s.version : null; }; module2.exports = clean3; @@ -27024,10 +27024,10 @@ var require_inc = __commonJS({ var require_diff = __commonJS({ "node_modules/semver/functions/diff.js"(exports2, module2) { "use strict"; - var parse3 = require_parse2(); + var parse4 = require_parse2(); var diff = (version1, version2) => { - const v1 = parse3(version1, null, true); - const v2 = parse3(version2, null, true); + const v1 = parse4(version1, null, true); + const v2 = parse4(version2, null, true); const comparison = v1.compare(v2); if (comparison === 0) { return null; @@ -27098,9 +27098,9 @@ var require_patch = __commonJS({ var require_prerelease = __commonJS({ "node_modules/semver/functions/prerelease.js"(exports2, module2) { "use strict"; - var parse3 = require_parse2(); + var parse4 = require_parse2(); var prerelease = (version, options) => { - const parsed = parse3(version, options); + const parsed = parse4(version, options); return parsed && parsed.prerelease.length ? parsed.prerelease : null; }; module2.exports = prerelease; @@ -27286,7 +27286,7 @@ var require_coerce = __commonJS({ "node_modules/semver/functions/coerce.js"(exports2, module2) { "use strict"; var SemVer = require_semver(); - var parse3 = require_parse2(); + var parse4 = require_parse2(); var { safeRe: re, t } = require_re(); var coerce3 = (version, options) => { if (version instanceof SemVer) { @@ -27321,7 +27321,7 @@ var require_coerce = __commonJS({ const patch = match2[4] || "0"; const prerelease = options.includePrerelease && match2[5] ? `-${match2[5]}` : ""; const build2 = options.includePrerelease && match2[6] ? `+${match2[6]}` : ""; - return parse3(`${major}.${minor}.${patch}${prerelease}${build2}`, options); + return parse4(`${major}.${minor}.${patch}${prerelease}${build2}`, options); }; module2.exports = coerce3; } @@ -27331,7 +27331,7 @@ var require_coerce = __commonJS({ var require_truncate = __commonJS({ "node_modules/semver/functions/truncate.js"(exports2, module2) { "use strict"; - var parse3 = require_parse2(); + var parse4 = require_parse2(); var constants = require_constants6(); var SemVer = require_semver(); var truncate = (version, truncation, options) => { @@ -27343,7 +27343,7 @@ var require_truncate = __commonJS({ }; var cloneInputVersion = (version, options) => { const versionStringToParse = version instanceof SemVer ? version.version : version; - return parse3(versionStringToParse, options); + return parse4(versionStringToParse, options); }; var doTruncation = (version, truncation) => { if (isPrerelease(truncation)) { @@ -28387,7 +28387,7 @@ var require_semver2 = __commonJS({ var constants = require_constants6(); var SemVer = require_semver(); var identifiers = require_identifiers(); - var parse3 = require_parse2(); + var parse4 = require_parse2(); var valid4 = require_valid(); var clean3 = require_clean(); var inc = require_inc(); @@ -28426,7 +28426,7 @@ var require_semver2 = __commonJS({ var simplifyRange = require_simplify(); var subset = require_subset(); module2.exports = { - parse: parse3, + parse: parse4, valid: valid4, clean: clean3, inc, @@ -32089,9 +32089,9 @@ var require_minimatch = __commonJS({ throw new TypeError("pattern is too long"); } }; - Minimatch2.prototype.parse = parse3; + Minimatch2.prototype.parse = parse4; var SUBPARSE = {}; - function parse3(pattern, isSub) { + function parse4(pattern, isSub) { assertValidPattern2(pattern); var options = this.options; if (pattern === "**") { @@ -33541,8 +33541,8 @@ var require_semver3 = __commonJS({ } } var i; - exports2.parse = parse3; - function parse3(version, options) { + exports2.parse = parse4; + function parse4(version, options) { if (!options || typeof options !== "object") { options = { loose: !!options, @@ -33570,12 +33570,12 @@ var require_semver3 = __commonJS({ } exports2.valid = valid4; function valid4(version, options) { - var v = parse3(version, options); + var v = parse4(version, options); return v ? v.version : null; } exports2.clean = clean3; function clean3(version, options) { - var s = parse3(version.trim().replace(/^[=v]+/, ""), options); + var s = parse4(version.trim().replace(/^[=v]+/, ""), options); return s ? s.version : null; } exports2.SemVer = SemVer; @@ -33811,8 +33811,8 @@ var require_semver3 = __commonJS({ if (eq(version1, version2)) { return null; } else { - var v1 = parse3(version1); - var v2 = parse3(version2); + var v1 = parse4(version1); + var v2 = parse4(version2); var prefix = ""; if (v1.prerelease.length || v2.prerelease.length) { prefix = "pre"; @@ -34518,7 +34518,7 @@ var require_semver3 = __commonJS({ } exports2.prerelease = prerelease; function prerelease(version, options) { - var parsed = parse3(version, options); + var parsed = parse4(version, options); return parsed && parsed.prerelease.length ? parsed.prerelease : null; } exports2.intersects = intersects; @@ -34555,7 +34555,7 @@ var require_semver3 = __commonJS({ if (match2 === null) { return null; } - return parse3(match2[2] + "." + (match2[3] || "0") + "." + (match2[4] || "0"), options); + return parse4(match2[2] + "." + (match2[3] || "0") + "." + (match2[4] || "0"), options); } } }); @@ -37255,7 +37255,7 @@ var require_ms = __commonJS({ options = options || {}; var type = typeof val; if (type === "string" && val.length > 0) { - return parse3(val); + return parse4(val); } else if (type === "number" && isFinite(val)) { return options.long ? fmtLong(val) : fmtShort(val); } @@ -37263,7 +37263,7 @@ var require_ms = __commonJS({ "val is not a non-empty string or a valid number. val=" + JSON.stringify(val) ); }; - function parse3(str) { + function parse4(str) { str = String(str); if (str.length > 100) { return; @@ -43199,7 +43199,7 @@ var require_deserializationPolicy = __commonJS({ return result; } async function deserializeResponseBody(jsonContentTypes, xmlContentTypes, response, options, parseXML) { - const parsedResponse = await parse3(jsonContentTypes, xmlContentTypes, response, options, parseXML); + const parsedResponse = await parse4(jsonContentTypes, xmlContentTypes, response, options, parseXML); if (!shouldDeserializeResponse(parsedResponse)) { return parsedResponse; } @@ -43300,7 +43300,7 @@ var require_deserializationPolicy = __commonJS({ } return { error: error3, shouldReturnResponse: false }; } - async function parse3(jsonContentTypes, xmlContentTypes, operationResponse, opts, parseXML) { + async function parse4(jsonContentTypes, xmlContentTypes, operationResponse, opts, parseXML) { if (!operationResponse.request.streamResponseStatusCodes?.has(operationResponse.status) && operationResponse.bodyAsText) { const text = operationResponse.bodyAsText; const contentType = operationResponse.headers.get("Content-Type") || ""; @@ -112628,7 +112628,7 @@ var require_dist_node2 = __commonJS({ return template.replace(/\/$/, ""); } } - function parse3(options) { + function parse4(options) { let method = options.method.toUpperCase(); let url2 = (options.url || "/").replace(/:([a-z]\w+)/g, "{$1}"); let headers = Object.assign({}, options.headers); @@ -112692,7 +112692,7 @@ var require_dist_node2 = __commonJS({ ); } function endpointWithDefaults2(defaults3, route, options) { - return parse3(merge2(defaults3, route, options)); + return parse4(merge2(defaults3, route, options)); } function withDefaults4(oldDefaults, newDefaults) { const DEFAULTS22 = merge2(oldDefaults, newDefaults); @@ -112701,7 +112701,7 @@ var require_dist_node2 = __commonJS({ DEFAULTS: DEFAULTS22, defaults: withDefaults4.bind(null, DEFAULTS22), merge: merge2.bind(null, DEFAULTS22), - parse: parse3 + parse: parse4 }); } var endpoint2 = withDefaults4(null, DEFAULTS2); @@ -116933,7 +116933,7 @@ var require_binary = __commonJS({ }); return stream2; }; - exports2.parse = function parse3(buffer) { + exports2.parse = function parse4(buffer) { var self2 = words(function(bytes, cb) { return function(name) { if (offset + bytes <= buffer.length) { @@ -151352,6 +151352,74 @@ function logMissingPerLanguageBundle({ logger }, language, location) { } // src/codeql-release.ts +function parseCodeQLReleaseUrl(input, apiDetails) { + let url2; + try { + url2 = new URL(input); + } catch { + return void 0; + } + if (url2.protocol !== "https:" || url2.username || url2.password) { + return void 0; + } + const isCurrentInstance = url2.origin === new URL(apiDetails.url).origin; + if (!isCurrentInstance && url2.origin !== new URL(GITHUB_DOTCOM_URL).origin) { + return void 0; + } + const match2 = url2.pathname.replace(/\/$/, "").match( + /^\/([\w.-]+)\/([\w.-]+)\/releases\/(?:tag\/(.+)|(codeql-bundle-[^/]+))$/ + ); + if (match2 === null) { + return void 0; + } + let tagName; + try { + tagName = decodeURIComponent(match2[3] ?? match2[4]); + } catch { + throw new ConfigurationError( + "Invalid URL encoding in the CodeQL release tag." + ); + } + return { + serverURL: url2.origin, + isCurrentInstance, + owner: match2[1], + repo: match2[2], + tagName + }; +} +function parseCliVersion(value) { + const parsed = semver7.parse(value); + if (parsed === null) { + return void 0; + } + return parsed.version + (parsed.build.length ? `+${parsed.build.join(".")}` : ""); +} +function getReleaseCliVersion(tagName, assetNames, logger) { + const versions = /* @__PURE__ */ new Set(); + for (const name of assetNames) { + const match2 = name.match(/^cli-version-(.+)\.txt$/); + if (match2 === null) { + continue; + } + const version = parseCliVersion(match2[1]); + if (version !== void 0) { + versions.add(version); + } else { + logger.debug(`Ignoring invalid CLI version marker ${name}.`); + } + } + if (versions.size > 1) { + logger.warning( + `Release ${tagName} has conflicting CLI version markers. Using a combined CodeQL bundle.` + ); + return void 0; + } + if (versions.size === 1) { + return versions.values().next().value; + } + return parseCliVersion(tagName.replace(/^codeql-bundle-/, "")); +} function encodeTag(tagName) { return tagName.split("/").map(encodeURIComponent).join("/"); } @@ -151368,7 +151436,8 @@ async function getRelease({ apiClient }, reference) { }); return { url: getReleasePageURL(reference), - getAssetURL: (name) => release2.assets.find((asset) => asset.name === name)?.url + getAssetURL: (name) => release2.assets.find((asset) => asset.name === name)?.url, + assetNames: release2.assets.map((asset) => asset.name) }; } function getPublicRelease(reference) { @@ -151378,6 +151447,21 @@ function getPublicRelease(reference) { getAssetURL: (name) => `${serverURL}/${owner}/${repo}/releases/download/${encodeTag(tagName)}/${name}` }; } +async function getRequestedRelease(action, requested) { + if (!requested.isCurrentInstance) { + return getPublicRelease(requested); + } + try { + return await getRelease(action, requested); + } catch (e) { + if (asHTTPError(e)?.status === 404) { + throw new ConfigurationError( + `Could not find the CodeQL release ${getReleasePageURL(requested)}. Check that it exists and that the token has access to it.` + ); + } + throw e; + } +} function getCompressionMethods({ cliVersion: cliVersion2, isLatestNightly, @@ -152145,8 +152229,7 @@ function getCodeQLActionRepository(logger) { } return getRequiredEnvParam("GITHUB_ACTION_REPOSITORY"); } -async function selectDefaultBundle(action, tagName, apiDetails, options) { - const { logger } = action; +function getDefaultBundleSources(apiDetails, logger) { const codeQLActionRepository = getCodeQLActionRepository(logger); const potentialDownloadSources = [ // This GitHub instance, and this Action. @@ -152156,12 +152239,25 @@ async function selectDefaultBundle(action, tagName, apiDetails, options) { // GitHub.com, and the canonical Action. [GITHUB_DOTCOM_URL, CODEQL_DEFAULT_ACTION_REPOSITORY] ]; - const uniqueDownloadSources = potentialDownloadSources.filter( - (source, index2, self2) => { - return !self2.slice(0, index2).some((other) => (0, import_fast_deep_equal.default)(source, other)); - } + return potentialDownloadSources.filter((source, index2, self2) => { + return !self2.slice(0, index2).some((other) => (0, import_fast_deep_equal.default)(source, other)); + }); +} +function isCacheableRelease(reference, cliVersion2, apiDetails, logger) { + if (cliVersion2 === void 0 || !/^\d+\.\d+\.\d+$/.test(cliVersion2) || reference.tagName !== `codeql-bundle-v${cliVersion2}`) { + return false; + } + const repository = `${reference.owner}/${reference.repo}`.toLowerCase(); + return getDefaultBundleSources(apiDetails, logger).some( + ([serverURL, sourceRepository]) => new URL(serverURL).origin === new URL(reference.serverURL).origin && sourceRepository.toLowerCase() === repository ); - for (const [serverURL, repository] of uniqueDownloadSources) { +} +async function selectDefaultBundle(action, tagName, apiDetails, options) { + const { logger } = action; + for (const [serverURL, repository] of getDefaultBundleSources( + apiDetails, + logger + )) { if (serverURL === GITHUB_DOTCOM_URL && repository === CODEQL_DEFAULT_ACTION_REPOSITORY) { break; } @@ -152347,11 +152443,14 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO toolsVersion: "local" }; } + const requestedRelease = toolsInput === void 0 ? void 0 : parseCodeQLReleaseUrl(toolsInput, apiDetails); let cliVersion2; let tagName; let url2; let bundle; - const canForceNightlyWithFF = isDynamicWorkflow() || isInTestMode(); + let release2; + let customReleaseURL; + const canForceNightlyWithFF = requestedRelease === void 0 && (isDynamicWorkflow() || isInTestMode()); const forceNightlyValueFF = await features.getValue("force_nightly" /* ForceNightly */); const forceNightly = forceNightlyValueFF && canForceNightlyWithFF; const nightlyRequestedByToolsInput = toolsInput !== void 0 && CODEQL_NIGHTLY_TOOLS_INPUTS.includes(toolsInput); @@ -152433,6 +152532,20 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO cliVersion2 = version.cliVersion; tagName = version.tagName; } + } else if (requestedRelease !== void 0) { + release2 = await getRequestedRelease( + { apiClient: getApiClient() }, + requestedRelease + ); + tagName = requestedRelease.tagName; + cliVersion2 = getReleaseCliVersion( + tagName, + release2.assetNames ?? [], + logger + ); + if (!isCacheableRelease(requestedRelease, cliVersion2, apiDetails, logger)) { + customReleaseURL = release2.url; + } } else if (toolsInput !== void 0) { tagName = tryGetTagNameFromUrl(toolsInput, logger); url2 = toolsInput; @@ -152453,68 +152566,21 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO cliVersion2 = version.cliVersion; tagName = version.tagName; } - const bundleVersion2 = tagName !== void 0 ? tryGetBundleVersionFromTagName(tagName, logger) : void 0; + const bundleVersion2 = ( + // Custom releases aren't cached, and their tags needn't contain a bundle version. + tagName !== void 0 && customReleaseURL === void 0 ? tryGetBundleVersionFromTagName(tagName, logger) : void 0 + ); const resolvedVersion = cliVersion2 ?? (bundleVersion2 !== void 0 ? convertToSemVer(bundleVersion2, logger) : void 0); const humanReadableVersion = resolvedVersion ?? tagName ?? url2 ?? "unknown"; logger.debug( `Attempting to obtain CodeQL tools. CLI version: ${cliVersion2 ?? "unknown"}, bundle tag name: ${tagName ?? "unknown"}, URL: ${url2 ?? "unspecified"}.` ); - let codeqlFolder; - if (cliVersion2) { - codeqlFolder = toolcache3.find("CodeQL", cliVersion2); - if (!codeqlFolder) { - logger.debug( - `Didn't find a version of the CodeQL tools in the toolcache with a version number exactly matching ${cliVersion2}.` - ); - const allVersions = toolcache3.findAllVersions("CodeQL"); - logger.debug( - `Found the following versions of the CodeQL tools in the toolcache: ${JSON.stringify( - allVersions - )}.` - ); - const candidateVersions = allVersions.filter( - (version) => version.startsWith(`${cliVersion2}-`) - ); - if (candidateVersions.length === 1) { - logger.debug( - `Exactly one version of the CodeQL tools starting with ${cliVersion2} found in the toolcache, using that.` - ); - codeqlFolder = toolcache3.find("CodeQL", candidateVersions[0]); - } else if (candidateVersions.length === 0) { - logger.debug( - `Didn't find any versions of the CodeQL tools starting with ${cliVersion2} in the toolcache. Trying next fallback method.` - ); - } else { - logger.warning( - `Found ${candidateVersions.length} versions of the CodeQL tools starting with ${cliVersion2} in the toolcache, but at most one was expected.` - ); - logger.debug("Trying next fallback method."); - } - } - } - if (!codeqlFolder && tagName) { - const fallbackVersion = await tryGetFallbackToolcacheVersion( - cliVersion2, - tagName, - logger - ); - if (fallbackVersion) { - codeqlFolder = toolcache3.find("CodeQL", fallbackVersion); - } else { - logger.debug( - `Could not determine a fallback toolcache version number for CodeQL tools version ${humanReadableVersion}.` - ); - } - } - if (codeqlFolder) { - logger.info( - `Found CodeQL tools version ${humanReadableVersion} in the toolcache.` - ); - } else { - logger.info( - `Did not find CodeQL tools version ${humanReadableVersion} in the toolcache.` - ); - } + const codeqlFolder = customReleaseURL === void 0 ? await findCodeQLInToolcache( + cliVersion2, + tagName, + humanReadableVersion, + logger + ) : void 0; if (codeqlFolder) { if (cliVersion2) { logger.info( @@ -152541,23 +152607,34 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO let compressionMethod; let perLanguageBundleFallback; if (!url2) { - if (tagName === void 0) { - throw new Error( - "Could not determine a release tag for the requested CodeQL bundle." + const selectionOptions = { + rawLanguages, + cliVersion: cliVersion2, + platform: getBundlePlatform(), + variant, + tarSupportsZstd + }; + let selection; + if (release2 !== void 0) { + selection = await selectBundle( + { env: getEnv(), features, logger }, + release2, + selectionOptions ); - } - ({ bundle, compressionMethod, perLanguageBundleFallback } = await selectDefaultBundle( - { env: getEnv(), features, logger }, - tagName, - apiDetails, - { - rawLanguages, - cliVersion: cliVersion2, - platform: getBundlePlatform(), - variant, - tarSupportsZstd + } else { + if (tagName === void 0) { + throw new Error( + "Could not determine a release tag for the requested CodeQL bundle." + ); } - )); + selection = await selectDefaultBundle( + { env: getEnv(), features, logger }, + tagName, + apiDetails, + selectionOptions + ); + } + ({ bundle, compressionMethod, perLanguageBundleFallback } = selection); url2 = bundle.url; } else { const method = inferCompressionMethod(url2); @@ -152584,11 +152661,71 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO bundleVersion: bundleVersion2, cliVersion: cliVersion2, compressionMethod, + ...customReleaseURL !== void 0 ? { customReleaseURL } : {}, ...perLanguageBundleFallback ? { perLanguageBundleFallback } : {}, sourceType: "download", toolsVersion: resolvedVersion ?? "unknown" }; } +async function findCodeQLInToolcache(cliVersion2, tagName, humanReadableVersion, logger) { + let codeqlFolder; + if (cliVersion2) { + codeqlFolder = toolcache3.find("CodeQL", cliVersion2); + if (!codeqlFolder) { + logger.debug( + `Didn't find a version of the CodeQL tools in the toolcache with a version number exactly matching ${cliVersion2}.` + ); + const allVersions = toolcache3.findAllVersions("CodeQL"); + logger.debug( + `Found the following versions of the CodeQL tools in the toolcache: ${JSON.stringify( + allVersions + )}.` + ); + const candidateVersions = allVersions.filter( + (version) => version.startsWith(`${cliVersion2}-`) + ); + if (candidateVersions.length === 1) { + logger.debug( + `Exactly one version of the CodeQL tools starting with ${cliVersion2} found in the toolcache, using that.` + ); + codeqlFolder = toolcache3.find("CodeQL", candidateVersions[0]); + } else if (candidateVersions.length === 0) { + logger.debug( + `Didn't find any versions of the CodeQL tools starting with ${cliVersion2} in the toolcache. Trying next fallback method.` + ); + } else { + logger.warning( + `Found ${candidateVersions.length} versions of the CodeQL tools starting with ${cliVersion2} in the toolcache, but at most one was expected.` + ); + logger.debug("Trying next fallback method."); + } + } + } + if (!codeqlFolder && tagName) { + const fallbackVersion = await tryGetFallbackToolcacheVersion( + cliVersion2, + tagName, + logger + ); + if (fallbackVersion) { + codeqlFolder = toolcache3.find("CodeQL", fallbackVersion); + } else { + logger.debug( + `Could not determine a fallback toolcache version number for CodeQL tools version ${humanReadableVersion}.` + ); + } + } + if (codeqlFolder) { + logger.info( + `Found CodeQL tools version ${humanReadableVersion} in the toolcache.` + ); + } else { + logger.info( + `Did not find CodeQL tools version ${humanReadableVersion} in the toolcache.` + ); + } + return codeqlFolder; +} async function tryGetFallbackToolcacheVersion(cliVersion2, tagName, logger) { const bundleVersion2 = tryGetBundleVersionFromTagName(tagName, logger); if (bundleVersion2 === void 0) { @@ -152648,6 +152785,11 @@ var downloadCodeQL = async function(source, apiDetails, tarVersion, tempDir, log }; }; function getToolcacheDestination({ logger }, source) { + if (source.customReleaseURL !== void 0) { + return new Failure( + `Not caching the CodeQL tools from ${source.customReleaseURL}, since we only cache stable releases in the CodeQL Action repositories.` + ); + } if (source.bundle.kind !== "combined") { return new Failure( "Not caching the CodeQL tools because they came from a bundle that contains only a single language." diff --git a/setup-codeql/action.yml b/setup-codeql/action.yml index 8d13eeaff0..b48a3f4413 100644 --- a/setup-codeql/action.yml +++ b/setup-codeql/action.yml @@ -10,6 +10,8 @@ inputs: - A local path to a CodeQL Bundle tarball, or - The URL of a CodeQL Bundle tarball GitHub release asset, or + - A release URL from GitHub.com or the current GitHub instance, such as + https://github.com/github/codeql-action/releases/tag/codeql-bundle-v2.27.0. - A special value `linked` which uses the version of the CodeQL tools that the Action has been bundled with. - A special value `nightly` which uses the latest nightly version of the diff --git a/src/codeql-bundle.ts b/src/codeql-bundle.ts index 99481834df..242d9270c4 100644 --- a/src/codeql-bundle.ts +++ b/src/codeql-bundle.ts @@ -29,6 +29,11 @@ export interface CodeQLDownloadSource { toolsVersion: string; /** The release lacks the eligible per-language bundle, so we selected the combined bundle. */ perLanguageBundleFallback?: true; + /** + * The page of a requested release that may contain a different build than the bundle we cache for + * its version, so we don't cache it. + */ + customReleaseURL?: string; } /** Returns the exact bundle asset name for a platform and optional language. */ diff --git a/src/codeql-release.test.ts b/src/codeql-release.test.ts index ce29a4bf13..abfcbbd93d 100644 --- a/src/codeql-release.test.ts +++ b/src/codeql-release.test.ts @@ -5,6 +5,9 @@ import { BundleSelectionOptions, getPublicRelease, getRelease, + getReleaseCliVersion, + getRequestedRelease, + parseCodeQLReleaseUrl, selectBundle, } from "./codeql-release"; import { ActionsEnvVars } from "./environment"; @@ -17,6 +20,7 @@ import { getTestEnv, initAllState, LoggedMessage, + SAMPLE_DOTCOM_API_DETAILS, } from "./testing-utils"; import { ConfigurationError, GitHubVariant } from "./util"; @@ -171,6 +175,37 @@ test("getRelease propagates API errors", async (t) => { t.deepEqual(fixture.requests, [fixture.releaseAPIURL]); }); +test("getRequestedRelease names a requested release that can't be found", async (t) => { + const requested = { ...REFERENCE, isCurrentInstance: true }; + await t.throwsAsync( + getRequestedRelease(releaseFixture({ status: 404 }).state, requested), + { + instanceOf: ConfigurationError, + message: `Could not find the CodeQL release ${RELEASE_PAGE}. Check that it exists and that the token has access to it.`, + }, + ); + t.like( + await t.throwsAsync( + getRequestedRelease(releaseFixture({ status: 500 }).state, requested), + ), + { status: 500 }, + ); +}); + +test("getRequestedRelease refers to a release on another instance by URL", async (t) => { + const fixture = releaseFixture(); + const release = await getRequestedRelease(fixture.state, { + ...REFERENCE, + isCurrentInstance: false, + }); + t.is( + release.getAssetURL(COMBINED), + `https://github.com/octo/tools/releases/download/${TAG}/${COMBINED}`, + ); + t.is(release.assetNames, undefined); + t.deepEqual(fixture.requests, []); +}); + test("getPublicRelease constructs download URLs without looking up the release", async (t) => { const fixture = releaseFixture(); const release = getPublicRelease({ ...REFERENCE, tagName: "nightly/v1+2" }); @@ -198,3 +233,122 @@ test("getPublicRelease constructs download URLs without looking up the release", ); t.deepEqual(fixture.requests, []); }); + +test("parseCodeQLReleaseUrl accepts web and legacy links and decodes tags once", (t) => { + for (const [suffix, tagName] of [ + [`tag/${TAG}`, TAG], + [TAG, TAG], + ["codeql-bundle-20230120", "codeql-bundle-20230120"], + ["codeql-bundle-v2.27.1-rc.1", "codeql-bundle-v2.27.1-rc.1"], + ["tag/run-123", "run-123"], + ["tag/build/123", "build/123"], + ["tag/build%2F123%2Brc%231", "build/123+rc#1"], + ["tag/build%252F123", "build%2F123"], + [`tag/${TAG}/?expanded=true#assets`, TAG], + ]) { + t.deepEqual( + parseCodeQLReleaseUrl( + `https://github.com/octo/tools/releases/${suffix}`, + SAMPLE_DOTCOM_API_DETAILS, + ), + { ...REFERENCE, isCurrentInstance: true, tagName }, + ); + } + t.throws( + () => + parseCodeQLReleaseUrl( + "https://github.com/octo/tools/releases/tag/%zz", + SAMPLE_DOTCOM_API_DETAILS, + ), + { instanceOf: ConfigurationError, message: /Invalid URL encoding/ }, + ); +}); + +test("parseCodeQLReleaseUrl matches the current instance and GitHub.com by origin", (t) => { + for (const [url, origin, serverURL] of [ + ["https://github.example.test", "https://github.com", "https://github.com"], + [ + "https://github.example.test/", + "https://github.example.test", + "https://github.example.test", + ], + [ + "https://GitHub.Example.test", + "https://github.example.test", + "https://github.example.test", + ], + [ + "https://github.example.test:443", + "https://GITHUB.example.test:443", + "https://github.example.test", + ], + ]) { + t.deepEqual( + parseCodeQLReleaseUrl(`${origin}/octo/tools/releases/tag/${TAG}`, { + auth: "token", + url, + apiURL: undefined, + }), + { + ...REFERENCE, + serverURL, + isCurrentInstance: serverURL !== "https://github.com", + }, + `${url} ${origin}`, + ); + } +}); + +test("parseCodeQLReleaseUrl excludes archives, REST references and untrusted URLs", (t) => { + for (const input of [ + "/tmp/codeql-bundle.tar.zst", + "nightly", + `https://github.com/octo/tools/releases/download/${TAG}/${JAVA}`, + "https://api.github.com/repos/octo/tools/releases/assets/123", + "https://api.github.com/repos/octo/tools/releases/123", + `https://api.github.com/repos/octo/tools/releases/tags/${TAG}`, + "https://github.com/octo/tools/releases/latest", + "https://github.com/octo/tools/releases/tag/", + "https://github.com/octo/tools/releases/run-123", + `https://github.com.example.test/octo/tools/releases/tag/${TAG}`, + `https://github.com:8443/octo/tools/releases/tag/${TAG}`, + `https://github.com@example.test/octo/tools/releases/tag/${TAG}`, + `https://user@github.com/octo/tools/releases/tag/${TAG}`, + `http://github.com/octo/tools/releases/tag/${TAG}`, + ]) { + t.is( + parseCodeQLReleaseUrl(input, SAMPLE_DOTCOM_API_DETAILS), + undefined, + input, + ); + } +}); + +test("getRelease returns the names of the release's assets", async (t) => { + const assetNames = [COMBINED, "cli-version-2.27.2.txt"]; + const fixture = releaseFixture({ assetNames }); + const release = await getRelease(fixture.state, REFERENCE); + t.deepEqual(release.assetNames, assetNames); +}); + +test("getReleaseCliVersion prefers an unambiguous marker asset to the tag", (t) => { + for (const [tagName, markers, cliVersion] of [ + [TAG, [], "2.27.1"], + [TAG, ["invalid"], "2.27.1"], + [TAG, ["2.27.2"], "2.27.2"], + [TAG, ["2.27.1", "2.28.0"], undefined], + ["codeql-bundle-20260101", [], undefined], + ["run-123", [], undefined], + ["run-123", ["2.27.2+202601011200"], "2.27.2+202601011200"], + ] as const) { + t.is( + getReleaseCliVersion( + tagName, + [COMBINED, ...markers.map((version) => `cli-version-${version}.txt`)], + getRecordingLogger([], { logToConsole: false }), + ), + cliVersion, + `${tagName} ${markers.join(",")}`, + ); + } +}); diff --git a/src/codeql-release.ts b/src/codeql-release.ts index 23f18503d3..6fad172d31 100644 --- a/src/codeql-release.ts +++ b/src/codeql-release.ts @@ -1,15 +1,22 @@ import * as semver from "semver"; import { ActionState } from "./action-common"; +import type { GitHubApiDetails } from "./api-client"; import { CodeQLBundle, getCodeQLBundleName } from "./codeql-bundle"; import { CODEQL_VERSION_ZSTD_BUNDLE } from "./feature-flags"; +import { Logger } from "./logging"; import { getPerLanguageBundleLanguage, logMissingPerLanguageBundle, } from "./per-language-bundles"; import { BundlePlatform } from "./platform"; import type { CompressionMethod } from "./tar"; -import { ConfigurationError, GitHubVariant } from "./util"; +import { + asHTTPError, + ConfigurationError, + GITHUB_DOTCOM_URL, + GitHubVariant, +} from "./util"; /** Identifies a release on a GitHub instance. */ export interface CodeQLReleaseReference { @@ -25,6 +32,111 @@ export interface CodeQLRelease { url: string; /** Returns the download URL for an asset, or `undefined` if the release doesn't have it. */ getAssetURL(name: string): string | undefined; + /** Names of the release's assets, if we looked up the release. */ + assetNames?: string[]; +} + +/** A release requested by URL, on this GitHub instance or on GitHub.com. */ +export interface RequestedRelease extends CodeQLReleaseReference { + isCurrentInstance: boolean; +} + +/** + * Recognizes release pages, including legacy bundle links, but never asset URLs. + * + * We only accept https URLs without credentials, on this GitHub instance, whose API we can use, or + * on GitHub.com, whose public releases we can download without credentials. The tag is + * percent-decoded once, and may contain `/`. + */ +export function parseCodeQLReleaseUrl( + input: string, + apiDetails: GitHubApiDetails, +): RequestedRelease | undefined { + let url: URL; + try { + url = new URL(input); + } catch { + return undefined; + } + if (url.protocol !== "https:" || url.username || url.password) { + return undefined; + } + + // GitHub instances are served from the root of their origin. + const isCurrentInstance = url.origin === new URL(apiDetails.url).origin; + if (!isCurrentInstance && url.origin !== new URL(GITHUB_DOTCOM_URL).origin) { + return undefined; + } + // Older links to bundle releases omit "tag/", which GitHub still supports. We only accept this + // form for bundle tags, since other names can clash with routes like `releases/latest`. + const match = url.pathname + .replace(/\/$/, "") + .match( + /^\/([\w.-]+)\/([\w.-]+)\/releases\/(?:tag\/(.+)|(codeql-bundle-[^/]+))$/, + ); + if (match === null) { + return undefined; + } + let tagName: string; + try { + tagName = decodeURIComponent(match[3] ?? match[4]); + } catch { + throw new ConfigurationError( + "Invalid URL encoding in the CodeQL release tag.", + ); + } + return { + serverURL: url.origin, + isCurrentInstance, + owner: match[1], + repo: match[2], + tagName, + }; +} + +function parseCliVersion(value: string): string | undefined { + const parsed = semver.parse(value); + if (parsed === null) { + return undefined; + } + // semver's normalized version omits build metadata, which distinguishes nightly builds. + return ( + parsed.version + (parsed.build.length ? `+${parsed.build.join(".")}` : "") + ); +} + +/** + * Returns the CLI version from the release's `cli-version-.txt` asset if it has one, and + * otherwise the version in the tag. Returns `undefined` if the release has conflicting markers. + */ +export function getReleaseCliVersion( + tagName: string, + assetNames: string[], + logger: Logger, +): string | undefined { + const versions = new Set(); + for (const name of assetNames) { + const match = name.match(/^cli-version-(.+)\.txt$/); + if (match === null) { + continue; + } + const version = parseCliVersion(match[1]); + if (version !== undefined) { + versions.add(version); + } else { + logger.debug(`Ignoring invalid CLI version marker ${name}.`); + } + } + if (versions.size > 1) { + logger.warning( + `Release ${tagName} has conflicting CLI version markers. Using a combined CodeQL bundle.`, + ); + return undefined; + } + if (versions.size === 1) { + return versions.values().next().value; + } + return parseCliVersion(tagName.replace(/^codeql-bundle-/, "")); } /** @@ -62,6 +174,7 @@ export async function getRelease( url: getReleasePageURL(reference), getAssetURL: (name) => release.assets.find((asset) => asset.name === name)?.url, + assetNames: release.assets.map((asset) => asset.name), }; } @@ -80,6 +193,32 @@ export function getPublicRelease( }; } +/** Gets a release requested by URL, looking it up if it's on the current GitHub instance. */ +export async function getRequestedRelease( + action: ActionState<["Api"]>, + requested: RequestedRelease, +): Promise { + if (!requested.isCurrentInstance) { + // We can't send the token to another instance, so only public releases are usable there, and we + // download them by URL rather than querying that instance's API. On the current instance we use + // the API instead, because release download URLs don't accept tokens and so would fail for + // private and internal repositories. Without the release's asset list, we assume the preferred + // compression is available and take the CLI version from the tag. + return getPublicRelease(requested); + } + try { + return await getRelease(action, requested); + } catch (e) { + if (asHTTPError(e)?.status === 404) { + throw new ConfigurationError( + `Could not find the CodeQL release ${getReleasePageURL(requested)}. Check that it ` + + "exists and that the token has access to it.", + ); + } + throw e; + } +} + /** Describes the job and runner that we are selecting a bundle for. */ export interface BundleSelectionOptions { /** Explicit `languages` input, which determines whether a per-language bundle is eligible. */ diff --git a/src/setup-codeql.test.ts b/src/setup-codeql.test.ts index d8d4cfe258..d8840694de 100644 --- a/src/setup-codeql.test.ts +++ b/src/setup-codeql.test.ts @@ -38,6 +38,7 @@ import { } from "./testing-utils"; import * as toolsDownload from "./tools-download"; import { + ConfigurationError, getErrorMessage, GitHubVariant, HTTPError, @@ -1382,6 +1383,361 @@ for (const scenario of ["per-language", "fallback", "missing"] as const) { ); } +const GHES_API_DETAILS = { + auth: "enterprise-token", + url: "https://github.example.test", + apiURL: "https://github.example.test/api/v3", +}; + +/** Models a hosted Linux runner with zstd and a release in `repository` on the instance `apiDetails` describes. */ +function stubRequestedRelease({ + apiDetails = SAMPLE_DOTCOM_API_DETAILS, + repository = "octo/tools", + tagName = `codeql-bundle-v${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}`, + assetNames = [ + "codeql-bundle-linux64.tar.zst", + "codeql-bundle-java-linux64.tar.zst", + ], + markers = [] as string[], + status = 200, +} = {}) { + sinon.stub(process, "platform").value("linux"); + sinon.stub(process, "arch").value("x64"); + sinon.stub(actionsUtil, "isRunningLocalAction").returns(false); + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted"; + sinon.stub(tar, "isZstdAvailable").resolves({ + available: true, + foundZstdBinary: true, + }); + const apiBase = `${apiDetails.apiURL}/repos/${repository}/releases`; + const assets = [ + ...assetNames, + ...markers.map((version) => `cli-version-${version}.txt`), + ].map((name, index) => ({ name, url: `${apiBase}/assets/${1000 + index}` })); + const requests: string[] = []; + const client = github.getOctokit("123", { + baseUrl: apiDetails.apiURL, + request: { + fetch: async (input) => { + const url = String(input); + requests.push(url); + if (url !== `${apiBase}/tags/${tagName}`) { + throw new Error(`Unexpected API request: ${url}`); + } + return new Response(JSON.stringify({ tag_name: tagName, assets }), { + status, + headers: { "content-type": "application/json" }, + }); + }, + }, + }); + sinon.stub(api, "getApiClient").value(() => client); + return { + assets, + requests, + releaseURL: `${apiDetails.url}/${repository}/releases/tag/${tagName}`, + }; +} + +for (const scenario of ["combined", "per-language", "fallback"] as const) { + test.serial( + `setupCodeQLBundle downloads a ${scenario} bundle from a custom release without using the toolcache`, + async (t) => { + const fixture = stubRequestedRelease(); + const extract = stubDownloadAndExtract(); + if (scenario === "fallback") { + extract.onFirstCall().rejects(new HTTPError("Not Found", 404)); + } + const find = sinon.spy(toolcache, "find"); + sinon.stub(actionsUtil, "isDynamicWorkflow").returns(true); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + createToolcacheEntry( + tmpDir, + "CodeQL", + MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION, + ); + const result = await setupCodeql.setupCodeQLBundle( + fixture.releaseURL, + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + PER_LANGUAGE_CLI_VERSION, + scenario === "combined" ? undefined : ["java"], + false, // useOverlayAwareDefaultCliVersion + // The requested release takes precedence over forcing the nightly. + createFeatures([Feature.PerLanguageBundles, Feature.ForceNightly]), + getRunnerLogger(true), + ); + + t.is(fixture.requests.length, 1); + t.true(find.notCalled); + t.is(result.toolsSource, setupCodeql.ToolsSource.Download); + t.is(result.toolsVersion, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION); + t.is(extract.callCount, scenario === "fallback" ? 2 : 1); + t.is( + extract.lastCall.args[0], + fixture.assets[scenario === "per-language" ? 1 : 0].url, + ); + t.is(extract.lastCall.args[3], "token token"); + t.is(path.dirname(result.codeqlFolder), tmpDir); + t.false(fs.existsSync(`${result.codeqlFolder}.complete`)); + }); + }, + ); +} + +test.serial( + "setupCodeQLBundle downloads the gzip bundle from a requested release that only has gzip bundles", + async (t) => { + const fixture = stubRequestedRelease({ + assetNames: ["codeql-bundle-linux64.tar.gz"], + }); + const extract = stubDownloadAndExtract(); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + const result = await setupCodeql.setupCodeQLBundle( + fixture.releaseURL, + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + PER_LANGUAGE_CLI_VERSION, + ["java"], + false, // useOverlayAwareDefaultCliVersion + createFeatures([Feature.PerLanguageBundles]), + getRunnerLogger(true), + ); + + t.true(extract.calledOnce); + t.is(extract.firstCall.args[0], fixture.assets[0].url); + t.is(extract.firstCall.args[1], "gzip"); + t.is(result.toolsSource, setupCodeql.ToolsSource.Download); + t.is(result.toolsDownloadStatusReport?.perLanguage, undefined); + }); + }, +); + +for (const [apiDetails, variant] of [ + [SAMPLE_DOTCOM_API_DETAILS, GitHubVariant.DOTCOM], + [GHES_API_DETAILS, GitHubVariant.GHES], +] as const) { + test.serial( + `setupCodeQLBundle caches a stable release in the CodeQL Action repository on ${variant}`, + async (t) => { + const fixture = stubRequestedRelease({ + apiDetails, + repository: "github/codeql-action", + }); + const extract = stubDownloadAndExtract(); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + const setup = () => + setupCodeql.setupCodeQLBundle( + fixture.releaseURL, + apiDetails, + tmpDir, + variant, + PER_LANGUAGE_CLI_VERSION, + undefined, // rawLanguages + false, // useOverlayAwareDefaultCliVersion + createFeatures([]), + getRunnerLogger(true), + ); + + const downloaded = await setup(); + t.deepEqual(fixture.requests, [ + `${apiDetails.apiURL}/repos/github/codeql-action/releases/tags/codeql-bundle-v${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}`, + ]); + t.is(downloaded.toolsSource, setupCodeql.ToolsSource.Download); + t.is(extract.firstCall.args[0], fixture.assets[0].url); + t.is(extract.firstCall.args[3], `token ${apiDetails.auth}`); + t.is( + downloaded.codeqlFolder, + toolcache.find("CodeQL", MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION), + ); + + const cached = await setup(); + t.is(cached.toolsSource, setupCodeql.ToolsSource.Toolcache); + t.is(cached.codeqlFolder, downloaded.codeqlFolder); + t.true(extract.calledOnce); + }); + }, + ); +} + +for (const { repository, tagName, markers } of [ + { + repository: "github/codeql-action", + tagName: "codeql-bundle-v2.27.1-rc.1", + markers: [], + }, + { + repository: "github/codeql-action", + tagName: "codeql-bundle-v2.27.1", + markers: ["2.27.1+202609241200"], + }, + { + repository: "github/codeql-action", + tagName: "codeql-bundle-20260924", + markers: ["2.27.1+202609241200"], + }, + { + repository: "github/codeql-action", + tagName: "codeql-bundle-acme-2.27.1", + markers: ["2.27.1"], + }, + { + repository: "octo/tools", + tagName: "codeql-bundle-feature_branch", + markers: [], + }, +]) { + const description = + markers.length === 0 + ? tagName + : `${tagName} with marker ${markers.join(", ")}`; + test.serial( + `setupCodeQLBundle doesn't share the toolcache with ${description} in ${repository}`, + async (t) => { + const fixture = stubRequestedRelease({ repository, tagName, markers }); + const extract = stubDownloadAndExtract(); + const find = sinon.spy(toolcache, "find"); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + createToolcacheEntry( + tmpDir, + "CodeQL", + MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION, + ); + const result = await setupCodeql.setupCodeQLBundle( + fixture.releaseURL, + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + PER_LANGUAGE_CLI_VERSION, + undefined, // rawLanguages + false, // useOverlayAwareDefaultCliVersion + createFeatures([]), + getRunnerLogger(true), + ); + + t.true(find.notCalled); + t.is(result.toolsSource, setupCodeql.ToolsSource.Download); + t.is(extract.firstCall.args[0], fixture.assets[0].url); + t.is(path.dirname(result.codeqlFolder), tmpDir); + t.false(fs.existsSync(`${result.codeqlFolder}.complete`)); + }); + }, + ); +} + +for (const repository of ["github/codeql-action", "octo/tools"]) { + test.serial( + `setupCodeQLBundle downloads a GitHub.com release in ${repository} from GHES by URL without credentials`, + async (t) => { + const fixture = stubRequestedRelease({ repository }); + const extract = stubDownloadAndExtract(); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + const result = await setupCodeql.setupCodeQLBundle( + fixture.releaseURL, + GHES_API_DETAILS, + tmpDir, + GitHubVariant.GHES, + PER_LANGUAGE_CLI_VERSION, + ["java"], + false, // useOverlayAwareDefaultCliVersion + createFeatures([Feature.PerLanguageBundles]), + getRunnerLogger(true), + ); + + t.deepEqual(fixture.requests, []); + t.true(extract.calledOnce); + t.is( + extract.firstCall.args[0], + `https://github.com/${repository}/releases/download/codeql-bundle-v${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}/codeql-bundle-linux64.tar.zst`, + ); + t.is(extract.firstCall.args[3], undefined); + t.is( + fs.existsSync(`${result.codeqlFolder}.complete`), + repository === "github/codeql-action", + ); + }); + }, + ); +} + +test.serial( + "setupCodeQLBundle doesn't substitute another release for a requested release that can't be found", + async (t) => { + const fixture = stubRequestedRelease({ status: 404 }); + const extract = stubDownloadAndExtract(); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + await t.throwsAsync( + setupCodeql.setupCodeQLBundle( + fixture.releaseURL, + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + PER_LANGUAGE_CLI_VERSION, + undefined, // rawLanguages + false, // useOverlayAwareDefaultCliVersion + createFeatures([]), + getRunnerLogger(true), + ), + { + instanceOf: ConfigurationError, + message: `Could not find the CodeQL release ${fixture.releaseURL}. Check that it exists and that the token has access to it.`, + }, + ); + t.is(fixture.requests.length, 1); + t.true(extract.notCalled); + }); + }, +); + +test.serial( + "setupCodeQLBundle uses the CLI version marker of a requested release", + async (t) => { + const fixture = stubRequestedRelease({ + tagName: "run-123", + markers: [MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION], + }); + const extract = stubDownloadAndExtract(); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + const result = await setupCodeql.setupCodeQLBundle( + fixture.releaseURL, + SAMPLE_DOTCOM_API_DETAILS, + tmpDir, + GitHubVariant.DOTCOM, + PER_LANGUAGE_CLI_VERSION, + ["java"], + false, // useOverlayAwareDefaultCliVersion + createFeatures([Feature.PerLanguageBundles]), + getRunnerLogger(true), + ); + + // The tag doesn't give a version, so the job can only use the per-language bundle because of + // the marker. + t.is(extract.firstCall.args[0], fixture.assets[1].url); + t.is(result.toolsVersion, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION); + t.is( + result.toolsDownloadStatusReport?.perLanguage?.tools_bundle_language, + BuiltInLanguage.java, + ); + }); + }, +); + for (const bundle of ["per-language", "combined", "fallback"] as const) { test.serial( `setupCodeQLBundle preserves the nightly version for a ${bundle} download`, diff --git a/src/setup-codeql.ts b/src/setup-codeql.ts index 98767143a4..47cd26686b 100644 --- a/src/setup-codeql.ts +++ b/src/setup-codeql.ts @@ -25,8 +25,13 @@ import { import { BundleSelection, BundleSelectionOptions, + CodeQLRelease, + CodeQLReleaseReference, getPublicRelease, getRelease, + getReleaseCliVersion, + getRequestedRelease, + parseCodeQLReleaseUrl, selectBundle, } from "./codeql-release"; import * as defaults from "./defaults.json"; @@ -89,6 +94,53 @@ export function getCodeQLActionRepository(logger: Logger): string { return util.getRequiredEnvParam("GITHUB_ACTION_REPOSITORY"); } +/** Returns the repositories that release the default bundles, most preferred first. */ +function getDefaultBundleSources( + apiDetails: api.GitHubApiDetails, + logger: Logger, +): Array<[serverURL: string, repository: string]> { + const codeQLActionRepository = getCodeQLActionRepository(logger); + const potentialDownloadSources: Array<[string, string]> = [ + // This GitHub instance, and this Action. + [apiDetails.url, codeQLActionRepository], + // This GitHub instance, and the canonical Action. + [apiDetails.url, CODEQL_DEFAULT_ACTION_REPOSITORY], + // GitHub.com, and the canonical Action. + [util.GITHUB_DOTCOM_URL, CODEQL_DEFAULT_ACTION_REPOSITORY], + ]; + // We now filter out any duplicates. + // Duplicates will happen either because the GitHub instance is GitHub.com, or because the Action is not a fork. + return potentialDownloadSources.filter((source, index, self) => { + return !self.slice(0, index).some((other) => deepEqual(source, other)); + }); +} + +/** + * Whether a release requested by URL contains the same build as the toolcache entry for its CLI + * version. The toolcache is keyed by version alone, so we only assume this for stable releases + * tagged `codeql-bundle-v` in the repositories we download the default bundles from. + */ +function isCacheableRelease( + reference: CodeQLReleaseReference, + cliVersion: string | undefined, + apiDetails: api.GitHubApiDetails, + logger: Logger, +): boolean { + if ( + cliVersion === undefined || + !/^\d+\.\d+\.\d+$/.test(cliVersion) || + reference.tagName !== `codeql-bundle-v${cliVersion}` + ) { + return false; + } + const repository = `${reference.owner}/${reference.repo}`.toLowerCase(); + return getDefaultBundleSources(apiDetails, logger).some( + ([serverURL, sourceRepository]) => + new URL(serverURL).origin === new URL(reference.serverURL).origin && + sourceRepository.toLowerCase() === repository, + ); +} + /** * Selects a bundle from the first release tagged `tagName` that has a compatible bundle, trying the * Action repositories on this GitHub instance before the canonical Action on GitHub.com. If we @@ -102,23 +154,10 @@ async function selectDefaultBundle( options: BundleSelectionOptions, ): Promise { const { logger } = action; - const codeQLActionRepository = getCodeQLActionRepository(logger); - const potentialDownloadSources = [ - // This GitHub instance, and this Action. - [apiDetails.url, codeQLActionRepository], - // This GitHub instance, and the canonical Action. - [apiDetails.url, CODEQL_DEFAULT_ACTION_REPOSITORY], - // GitHub.com, and the canonical Action. - [util.GITHUB_DOTCOM_URL, CODEQL_DEFAULT_ACTION_REPOSITORY], - ]; - // We now filter out any duplicates. - // Duplicates will happen either because the GitHub instance is GitHub.com, or because the Action is not a fork. - const uniqueDownloadSources = potentialDownloadSources.filter( - (source, index, self) => { - return !self.slice(0, index).some((other) => deepEqual(source, other)); - }, - ); - for (const [serverURL, repository] of uniqueDownloadSources) { + for (const [serverURL, repository] of getDefaultBundleSources( + apiDetails, + logger, + )) { // If we've reached the final case, short-circuit the API check since we know the bundle exists and is public. if ( serverURL === util.GITHUB_DOTCOM_URL && @@ -191,6 +230,12 @@ export function tryGetTagNameFromUrl( return match[1]; } +/** + * Converts a bundle version to a semantic version, for example to use in the toolcache. Semantic + * versions are normalized with `semver.clean`, which drops a leading `v` and any build metadata. + * Anything else, such as a date, becomes a prerelease of `0.0.0`. Throws if the result isn't a + * valid semantic version. + */ export function convertToSemVer(version: string, logger: Logger): string { if (!semver.valid(version)) { logger.debug( @@ -386,6 +431,11 @@ async function resolveDefaultCliVersion( * We handle the `tools` input in this order: * * - A local path is extracted without using the toolcache. + * - A release URL selects a bundle from that release, and takes precedence over the `force_nightly` + * feature flag. Only stable releases in the repositories we download the default bundles from use + * the toolcache, since other releases may contain a different build than the cached bundle for + * their version. Bundle URLs keep using the toolcache for the version in their tag, for + * compatibility. * - `nightly` or `nightly-latest`, or the `force_nightly` feature flag in a dynamic workflow, * selects a bundle from the latest nightly release. We then continue with that bundle's URL. * - `linked`, or its old name `latest`, selects the version shipped with the Action. @@ -394,8 +444,9 @@ async function resolveDefaultCliVersion( * - Any other value is the URL of a bundle. * - Without a `tools` input, we use the default version. * - * Apart from a local path, we look for the resolved version in the toolcache before downloading. A - * cached version takes precedence even if the job could use a per-language bundle. + * For other inputs apart from a local path, we look for the resolved version in the toolcache + * before downloading. A cached version takes precedence even if the job could use a per-language + * bundle. * * @param toolsInput The argument provided for the `tools` input, if any. * @param defaultCliVersion The default CLI version that's linked to the CodeQL Action. @@ -444,6 +495,11 @@ export async function getCodeQLSource( }; } + const requestedRelease = + toolsInput === undefined + ? undefined + : parseCodeQLReleaseUrl(toolsInput, apiDetails); + /** Requested CLI version number, for example 2.12.6. */ let cliVersion: string | undefined; /** Tag name of the CodeQL bundle, for example `codeql-bundle-20230120`. */ @@ -455,10 +511,17 @@ export async function getCodeQLSource( */ let url: string | undefined; let bundle: CodeQLBundle | undefined; + /** The release requested by URL, which we select the bundle from. */ + let release: CodeQLRelease | undefined; + /** The page of a requested release whose bundles we don't cache. */ + let customReleaseURL: string | undefined; // We allow forcing the nightly CLI via the FF for `dynamic` events (or in test mode) where the - // `tools` input cannot be adjusted to explicitly request it. - const canForceNightlyWithFF = isDynamicWorkflow() || util.isInTestMode(); + // `tools` input cannot be adjusted to explicitly request it. An explicitly requested release + // takes precedence. + const canForceNightlyWithFF = + requestedRelease === undefined && + (isDynamicWorkflow() || util.isInTestMode()); const forceNightlyValueFF = await features.getValue(Feature.ForceNightly); const forceNightly = forceNightlyValueFF && canForceNightlyWithFF; @@ -579,6 +642,20 @@ export async function getCodeQLSource( cliVersion = version.cliVersion; tagName = version.tagName; } + } else if (requestedRelease !== undefined) { + release = await getRequestedRelease( + { apiClient: api.getApiClient() }, + requestedRelease, + ); + tagName = requestedRelease.tagName; + cliVersion = getReleaseCliVersion( + tagName, + release.assetNames ?? [], + logger, + ); + if (!isCacheableRelease(requestedRelease, cliVersion, apiDetails, logger)) { + customReleaseURL = release.url; + } } else if (toolsInput !== undefined) { // Any other value is a bundle URL, including one we selected from the latest nightly above. // We use the version in its tag, if any, for the toolcache, so we assume that bundles with the @@ -606,7 +683,8 @@ export async function getCodeQLSource( } const bundleVersion = - tagName !== undefined + // Custom releases aren't cached, and their tags needn't contain a bundle version. + tagName !== undefined && customReleaseURL === undefined ? tryGetBundleVersionFromTagName(tagName, logger) : undefined; const resolvedVersion = @@ -623,77 +701,16 @@ export async function getCodeQLSource( `URL: ${url ?? "unspecified"}.`, ); - let codeqlFolder: string | undefined; - - if (cliVersion) { - // If we find the specified CLI version, we always use that. - codeqlFolder = toolcache.find("CodeQL", cliVersion); - - // Fall back to matching `x.y.z-`. - if (!codeqlFolder) { - logger.debug( - "Didn't find a version of the CodeQL tools in the toolcache with a version number " + - `exactly matching ${cliVersion}.`, - ); - const allVersions = toolcache.findAllVersions("CodeQL"); - logger.debug( - `Found the following versions of the CodeQL tools in the toolcache: ${JSON.stringify( - allVersions, - )}.`, - ); - // If there is exactly one version of the CodeQL tools in the toolcache, and that version is - // the form `x.y.z-`, then use it. - const candidateVersions = allVersions.filter((version) => - version.startsWith(`${cliVersion}-`), - ); - if (candidateVersions.length === 1) { - logger.debug( - `Exactly one version of the CodeQL tools starting with ${cliVersion} found in the ` + - "toolcache, using that.", - ); - codeqlFolder = toolcache.find("CodeQL", candidateVersions[0]); - } else if (candidateVersions.length === 0) { - logger.debug( - `Didn't find any versions of the CodeQL tools starting with ${cliVersion} ` + - `in the toolcache. Trying next fallback method.`, - ); - } else { - logger.warning( - `Found ${candidateVersions.length} versions of the CodeQL tools starting with ` + - `${cliVersion} in the toolcache, but at most one was expected.`, - ); - logger.debug("Trying next fallback method."); - } - } - } - - // Fall back to matching `0.0.0-`. - if (!codeqlFolder && tagName) { - const fallbackVersion = await tryGetFallbackToolcacheVersion( - cliVersion, - tagName, - logger, - ); - if (fallbackVersion) { - codeqlFolder = toolcache.find("CodeQL", fallbackVersion); - } else { - logger.debug( - "Could not determine a fallback toolcache version number for CodeQL tools version " + - `${humanReadableVersion}.`, - ); - } - } - - if (codeqlFolder) { - logger.info( - `Found CodeQL tools version ${humanReadableVersion} in the toolcache.`, - ); - } else { - logger.info( - `Did not find CodeQL tools version ${humanReadableVersion} in the toolcache.`, - ); - } - + // A custom release may contain a different build than the bundle with the same version. + const codeqlFolder = + customReleaseURL === undefined + ? await findCodeQLInToolcache( + cliVersion, + tagName, + humanReadableVersion, + logger, + ) + : undefined; if (codeqlFolder) { if (cliVersion) { logger.info( @@ -730,24 +747,34 @@ export async function getCodeQLSource( let perLanguageBundleFallback: true | undefined; if (!url) { - if (tagName === undefined) { - throw new Error( - "Could not determine a release tag for the requested CodeQL bundle.", + const selectionOptions: BundleSelectionOptions = { + rawLanguages, + cliVersion, + platform: getBundlePlatform(), + variant, + tarSupportsZstd, + }; + let selection: BundleSelection; + if (release !== undefined) { + selection = await selectBundle( + { env: getEnv(), features, logger }, + release, + selectionOptions, ); - } - ({ bundle, compressionMethod, perLanguageBundleFallback } = - await selectDefaultBundle( + } else { + if (tagName === undefined) { + throw new Error( + "Could not determine a release tag for the requested CodeQL bundle.", + ); + } + selection = await selectDefaultBundle( { env: getEnv(), features, logger }, tagName, apiDetails, - { - rawLanguages, - cliVersion, - platform: getBundlePlatform(), - variant, - tarSupportsZstd, - }, - )); + selectionOptions, + ); + } + ({ bundle, compressionMethod, perLanguageBundleFallback } = selection); url = bundle.url; } else { const method = tar.inferCompressionMethod(url); @@ -779,12 +806,97 @@ export async function getCodeQLSource( bundleVersion, cliVersion, compressionMethod, + ...(customReleaseURL !== undefined ? { customReleaseURL } : {}), ...(perLanguageBundleFallback ? { perLanguageBundleFallback } : {}), sourceType: "download", toolsVersion: resolvedVersion ?? "unknown", }; } +/** + * Looks for the requested version of the CodeQL tools in the toolcache, allowing for the different + * version numbers that toolcaches may use for the same bundle. We try the exact CLI version, then a + * single `x.y.z-*` entry for that version, then `0.0.0-`. The `x.y.z-*` entry can + * be any prerelease of the CLI version. + */ +async function findCodeQLInToolcache( + cliVersion: string | undefined, + tagName: string | undefined, + humanReadableVersion: string, + logger: Logger, +): Promise { + let codeqlFolder: string | undefined; + + if (cliVersion) { + // If we find the specified CLI version, we always use that. + codeqlFolder = toolcache.find("CodeQL", cliVersion); + + // Fall back to a single `x.y.z-*` entry, since older toolcaches store bundles as + // `x.y.z-`. + if (!codeqlFolder) { + logger.debug( + "Didn't find a version of the CodeQL tools in the toolcache with a version number " + + `exactly matching ${cliVersion}.`, + ); + const allVersions = toolcache.findAllVersions("CodeQL"); + logger.debug( + `Found the following versions of the CodeQL tools in the toolcache: ${JSON.stringify( + allVersions, + )}.`, + ); + const candidateVersions = allVersions.filter((version) => + version.startsWith(`${cliVersion}-`), + ); + if (candidateVersions.length === 1) { + logger.debug( + `Exactly one version of the CodeQL tools starting with ${cliVersion} found in the ` + + "toolcache, using that.", + ); + codeqlFolder = toolcache.find("CodeQL", candidateVersions[0]); + } else if (candidateVersions.length === 0) { + logger.debug( + `Didn't find any versions of the CodeQL tools starting with ${cliVersion} ` + + `in the toolcache. Trying next fallback method.`, + ); + } else { + logger.warning( + `Found ${candidateVersions.length} versions of the CodeQL tools starting with ` + + `${cliVersion} in the toolcache, but at most one was expected.`, + ); + logger.debug("Trying next fallback method."); + } + } + } + + // Fall back to matching `0.0.0-`. + if (!codeqlFolder && tagName) { + const fallbackVersion = await tryGetFallbackToolcacheVersion( + cliVersion, + tagName, + logger, + ); + if (fallbackVersion) { + codeqlFolder = toolcache.find("CodeQL", fallbackVersion); + } else { + logger.debug( + "Could not determine a fallback toolcache version number for CodeQL tools version " + + `${humanReadableVersion}.`, + ); + } + } + + if (codeqlFolder) { + logger.info( + `Found CodeQL tools version ${humanReadableVersion} in the toolcache.`, + ); + } else { + logger.info( + `Did not find CodeQL tools version ${humanReadableVersion} in the toolcache.`, + ); + } + return codeqlFolder; +} + /** * Gets a fallback version number to use when looking for CodeQL in the toolcache if we didn't find * the `x.y.z` version. This is to support old versions of the toolcache. @@ -878,11 +990,21 @@ export const downloadCodeQL = async function ( /** * Returns the canonical toolcache directory, or the reason the bundle cannot be cached. + * + * The toolcache is keyed by version, so we don't cache bundles that would give a later request for + * the same version the wrong tools: per-language bundles, which lack the other languages, and + * custom releases, which may be a different build. */ function getToolcacheDestination( { logger }: ActionState<["Logger"]>, source: CodeQLDownloadSource, ): util.Result { + if (source.customReleaseURL !== undefined) { + return new util.Failure( + `Not caching the CodeQL tools from ${source.customReleaseURL}, since we only cache stable ` + + "releases in the CodeQL Action repositories.", + ); + } if (source.bundle.kind !== "combined") { return new util.Failure( "Not caching the CodeQL tools because they came from a bundle that contains only a " + @@ -973,11 +1095,11 @@ function getCanonicalToolcacheVersion( bundleVersion: string, logger: Logger, ): string { - // If the CLI version is a pre-release or contains build metadata, then cache the - // bundle as `0.0.0-` to avoid the bundle being interpreted as containing a stable - // CLI release. In principle, it should be enough to just check that the CLI version isn't a - // pre-release, but the version numbers of CodeQL nightlies have the format `x.y.z+`, - // and we don't want these nightlies to override stable CLI versions in the toolcache. + // If the CLI version is unknown, as for nightlies, which are tagged by date, or isn't a plain + // `x.y.z`, for example a prerelease, cache the bundle under its bundle version, such as + // `0.0.0-` or `x.y.z-rc.1`, so that it isn't cached as a stable release. However, + // `convertToSemVer` drops build metadata, so a bundle URL tagged `codeql-bundle-vX.Y.Z+` + // is still cached as `X.Y.Z`. if (!cliVersion?.match(/^[0-9]+\.[0-9]+\.[0-9]+$/)) { return convertToSemVer(bundleVersion, logger); }