From 0692d8034d6ab4e09429fa3fc711d39c64fb01e9 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sun, 4 Oct 2026 23:52:15 +0300 Subject: [PATCH 1/2] feat: add --last-failed option to run and run-workers Every run and run-workers now saves result.json to the output dir via Result.save(). --last-failed reads it and runs only tests whose state is failed, matched by test uid, filtering before tests are split into workers. Co-Authored-By: Claude Opus 5.5 --- bin/codecept.js | 2 + docs/commands.md | 30 ++++ lib/codecept.js | 18 +++ lib/command/run-workers.js | 8 + lib/command/run.js | 5 +- lib/command/utils.js | 15 ++ lib/command/workers/runTests.js | 1 + lib/result.js | 16 ++ lib/workers.js | 4 + .../configs/last-failed/codecept.conf.js | 9 ++ .../sandbox/configs/last-failed/math_test.js | 22 +++ .../configs/last-failed/strings_test.js | 13 ++ test/runner/last_failed_test.js | 140 ++++++++++++++++++ 13 files changed, 281 insertions(+), 2 deletions(-) create mode 100644 test/data/sandbox/configs/last-failed/codecept.conf.js create mode 100644 test/data/sandbox/configs/last-failed/math_test.js create mode 100644 test/data/sandbox/configs/last-failed/strings_test.js create mode 100644 test/runner/last_failed_test.js diff --git a/bin/codecept.js b/bin/codecept.js index e270e2920..529db7314 100755 --- a/bin/codecept.js +++ b/bin/codecept.js @@ -178,6 +178,7 @@ program .option('-p, --plugins ', 'enable plugins, comma-separated') .option('--shuffle', 'Shuffle the order in which test files run') .option('--shard ', 'run only a fraction of tests (e.g., --shard 1/4)') + .option('--last-failed', 'run only tests that failed in the previous run') // mocha options .option('--colors', 'force enabling of colors') @@ -207,6 +208,7 @@ program .option(commandFlags.config.flag, commandFlags.config.description) .option('-g, --grep ', 'only run tests matching ') .option('-i, --invert', 'inverts --grep matches') + .option('--last-failed', 'run only tests that failed in the previous run') .option('-o, --override [value]', 'override current config options') .option('--suites', 'parallel execution of suites not single tests') .option('--by ', 'test distribution strategy: "test" (pre-assign individual tests), "suite" (pre-assign test suites), or "pool" (dynamic distribution for optimal load balancing, recommended)') diff --git a/docs/commands.md b/docs/commands.md index dff6a23b6..015248201 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -53,6 +53,14 @@ Run test files in shuffled order npx codeceptjs run --shuffle ``` +Rerun only tests that failed in the previous run + +```sh +npx codeceptjs run --last-failed +``` + +See [Last Failed](#last-failed) for details. + Run single test in debug mode (see more in [debugging](#Debugging) section) ```sh @@ -157,6 +165,9 @@ npx codeceptjs run-workers 3 --by suite # Pool mode with filtering npx codeceptjs run-workers 4 --by pool --grep "@smoke" + +# Rerun only tests that failed in the previous run +npx codeceptjs run-workers 3 --last-failed ``` **Test Distribution Strategies:** @@ -169,6 +180,25 @@ The pool mode provides the best load balancing by maintaining tests in a shared See [Parallel Execution](/parallel) documentation for more details. +## Last Failed + +Rerun only the tests that failed in the previous run. Works with `run` and `run-workers`, including all `--by` strategies: + +```sh +npx codeceptjs run --last-failed +npx codeceptjs run-workers 3 --last-failed +``` + +Every `run` and `run-workers` writes a report to `result.json` in the output directory (`output` in config). It lists each executed test with its state. `--last-failed` reads this report and runs only the tests whose state is `failed`: + +- Skipped and pending tests are not considered failed. +- Tests are matched by their title, including the Feature title. A renamed test or Feature is not matched. +- Combined with `--grep`, only failed tests that also match the pattern are run. +- The report always reflects the most recent run, including a `--last-failed` run. Tests that pass in a rerun drop out of the next `--last-failed` set; tests that did not run (filtered out by `--grep`, sharding, and so on) are not included. +- If the report does not exist, the command fails with "No previous run found". If the last run had no failures, it prints "No failed tests in the last run" and exits with code 0. + +`--last-failed` is different from [run-rerun](#run-rerun), which repeats whole runs to detect flaky tests. + ## Run Rerun Run tests multiple times to detect and fix flaky tests. diff --git a/lib/codecept.js b/lib/codecept.js index 6c10bca08..19022b7d7 100644 --- a/lib/codecept.js +++ b/lib/codecept.js @@ -23,6 +23,7 @@ import loadTests from './mocha/loadTests.js' import { validateTypeScriptSetup, getTSNodeESMWarning } from './utils/loaderCheck.js' import recorder from './recorder.js' import store from './store.js' +import Result from './result.js' import storeListener from './listener/store.js' import stepsListener from './listener/steps.js' @@ -264,6 +265,21 @@ class Codecept { return testFiles.slice(startIndex, endIndex) } + /** + * Keeps only tests that failed in the previous run when `--last-failed` is set. + * + * @param {Mocha} mocha + */ + filterLastFailedTests(mocha) { + if (!this.opts.lastFailed) return + const failedTestUids = new Set(Result.loadFailedTests().map(test => test.uid)) + const filterSuite = suite => { + suite.tests = suite.tests.filter(test => failedTestUids.has(test.uid)) + suite.suites.forEach(filterSuite) + } + filterSuite(mocha.suite) + } + /** * Run a specific test or all loaded tests. * @@ -317,6 +333,7 @@ class Codecept { // Worker threads pre-load and filter tests before run(); reloading duplicates Gherkin suites. if (!mocha.suite.total()) { await loadTests(mocha) + this.filterLastFailedTests(mocha) } else { mocha.lazyLoadFiles(true) } @@ -327,6 +344,7 @@ class Codecept { event.emit(event.all.after, this) // Wait for any recorder tasks added by event.all.after handlers await recorder.promise() + if (!this.opts.child && !store.dryRun) container.result().save() // Set exit code based on test failures if (failures) { process.exitCode = 1 diff --git a/lib/command/run-workers.js b/lib/command/run-workers.js index 00c343532..cf1f17037 100644 --- a/lib/command/run-workers.js +++ b/lib/command/run-workers.js @@ -5,7 +5,9 @@ import store from '../store.js' import event from '../event.js' import Workers from '../workers.js' import Codecept from '../codecept.js' +import Config from '../config.js' import { getMachineInfo } from './info.js' +import { getConfig, getTestRoot, createOutputDir, hasLastFailedTests } from './utils.js' export default async function (workerCount, selectedRuns, options) { process.env.profile = options.profile @@ -36,6 +38,12 @@ export default async function (workerCount, selectedRuns, options) { selectedRuns, } + if (options.lastFailed) { + await getConfig(testConfig) + const outputDir = createOutputDir(Config.append(overrideConfigs), getTestRoot(testConfig)) + if (!(await hasLastFailedTests(outputDir))) return + } + const numberOfWorkers = parseInt(workerCount, 10) output.print(`CodeceptJS v${Codecept.version()} ${output.standWithUkraine()}`) diff --git a/lib/command/run.js b/lib/command/run.js index 8a54c10d1..737ef97db 100644 --- a/lib/command/run.js +++ b/lib/command/run.js @@ -1,4 +1,4 @@ -import { getConfig, printError, getTestRoot, createOutputDir, autoExit } from './utils.js' +import { getConfig, printError, getTestRoot, createOutputDir, autoExit, hasLastFailedTests } from './utils.js' import Config from '../config.js' import store from '../store.js' import Codecept from '../codecept.js' @@ -21,7 +21,8 @@ export default async function (test, options) { config = Config.append(JSON.parse(options.override)) } const testRoot = getTestRoot(configFile) - createOutputDir(config, testRoot) + const outputDir = createOutputDir(config, testRoot) + if (options.lastFailed && !(await hasLastFailedTests(outputDir))) return const codecept = new Codecept(config, options) diff --git a/lib/command/utils.js b/lib/command/utils.js index 49b1467a7..1192c1f60 100644 --- a/lib/command/utils.js +++ b/lib/command/utils.js @@ -105,6 +105,21 @@ export const createOutputDir = (config, testRoot) => { output.print(`creating output directory: ${outputDir}`) mkdirp.sync(outputDir) } + return outputDir +} + +export async function hasLastFailedTests(outputDir) { + const { default: Result } = await import('../result.js') + let failedTests + try { + failedTests = Result.loadFailedTests(outputDir) + } catch (err) { + output.error(err.message) + process.exitCode = 1 + return false + } + if (!failedTests.length) output.print('No failed tests in the last run') + return failedTests.length > 0 } export async function autoExit() { diff --git a/lib/command/workers/runTests.js b/lib/command/workers/runTests.js index 9d6519117..c9931ace3 100644 --- a/lib/command/workers/runTests.js +++ b/lib/command/workers/runTests.js @@ -287,6 +287,7 @@ async function runPoolTests() { // Load only the assigned test file mocha.files = [testIdentifier] await loadTests(mocha) + codecept.filterLastFailedTests(mocha) if (mocha.suite.total() > 0) { // Run only the tests in the current mocha suite diff --git a/lib/result.js b/lib/result.js index c7ff4f587..8fbe5cc3e 100644 --- a/lib/result.js +++ b/lib/result.js @@ -216,6 +216,22 @@ class Result { fs.writeFileSync(path.join(store.outputDir, fileName), JSON.stringify(this.simplify(), null, 2)) } + /** + * Loads tests that failed in a result saved by a previous run. + * + * @param {string} [outputDir] Directory containing the saved result. Defaults to `output_dir`. + * @param {string} [fileName] Path to the JSON file, relative to `outputDir`. Defaults to "result.json". + * @returns {Array<{uid: string, title: string, state: string}>} Serialized failed tests. + */ + static loadFailedTests(outputDir = store.outputDir, fileName = 'result.json') { + const resultFile = path.join(outputDir, fileName) + if (!fs.existsSync(resultFile)) { + throw new Error(`No previous run found (${resultFile} does not exist), run tests once first`) + } + const { tests = [] } = JSON.parse(fs.readFileSync(resultFile, 'utf8')) + return tests.filter(test => test.state === 'failed') + } + /** * Adds stats to this result. * diff --git a/lib/workers.js b/lib/workers.js index 84a96b02d..df0300765 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -375,6 +375,7 @@ class Workers extends EventEmitter { const mocha = Container.mocha() mocha.files = files await loadTests(mocha) + this.codecept.filterLastFailedTests(mocha) const groups = populateGroups(numberOfWorkers) let groupCounter = 0 @@ -463,6 +464,7 @@ class Workers extends EventEmitter { const mocha = Container.mocha() mocha.files = files await loadTests(mocha) + this.codecept.filterLastFailedTests(mocha) mocha.suite.suites.forEach(suite => { const i = indexOfSmallestElement(groups) @@ -802,6 +804,7 @@ class Workers extends EventEmitter { // For tests with retries configured, emit all failures + final success // For tests without retries, emit only final state const lastState = states[states.length - 1] + Container.result().addTest(deserializeTest(lastState.data)) // Check if this test had retries by looking for failure followed by success const hasRetryPattern = states.length > 1 && states.some((s, i) => s.isFailed && i < states.length - 1 && !states[i + 1].isFailed) @@ -829,6 +832,7 @@ class Workers extends EventEmitter { this.emit(event.all.result, Container.result()) event.dispatcher.emit(event.workers.result, Container.result()) + Container.result().save() this.emit('end') // internal event } diff --git a/test/data/sandbox/configs/last-failed/codecept.conf.js b/test/data/sandbox/configs/last-failed/codecept.conf.js new file mode 100644 index 000000000..5df0d76ac --- /dev/null +++ b/test/data/sandbox/configs/last-failed/codecept.conf.js @@ -0,0 +1,9 @@ +export const config = { + tests: './*_test.js', + output: './output', + helpers: {}, + include: {}, + bootstrap: null, + mocha: {}, + name: 'last-failed', +} diff --git a/test/data/sandbox/configs/last-failed/math_test.js b/test/data/sandbox/configs/last-failed/math_test.js new file mode 100644 index 000000000..d1ba1ae8c --- /dev/null +++ b/test/data/sandbox/configs/last-failed/math_test.js @@ -0,0 +1,22 @@ +import assert from 'assert' + +Feature('Math') + +Scenario('adds numbers', () => { + console.log('executed: adds numbers') + assert.equal(1 + 1, 2) +}) + +Scenario('divides numbers @smoke', () => { + console.log('executed: divides numbers') + assert.equal(4 / 2, 3) +}) + +Scenario('multiplies numbers', () => { + console.log('executed: multiplies numbers') + assert.equal(2 * 2, process.env.LAST_FAILED_FIXED ? 4 : 5) +}) + +xScenario('subtracts numbers', () => { + console.log('executed: subtracts numbers') +}) diff --git a/test/data/sandbox/configs/last-failed/strings_test.js b/test/data/sandbox/configs/last-failed/strings_test.js new file mode 100644 index 000000000..db28988f0 --- /dev/null +++ b/test/data/sandbox/configs/last-failed/strings_test.js @@ -0,0 +1,13 @@ +import assert from 'assert' + +Feature('Strings') + +Scenario('concats strings', () => { + console.log('executed: concats strings') + assert.equal('a' + 'b', 'ab') +}) + +Scenario('uppercases strings @smoke', () => { + console.log('executed: uppercases strings') + assert.equal('a'.toUpperCase(), 'a') +}) diff --git a/test/runner/last_failed_test.js b/test/runner/last_failed_test.js new file mode 100644 index 000000000..b0a44165d --- /dev/null +++ b/test/runner/last_failed_test.js @@ -0,0 +1,140 @@ +import { expect } from 'expect' +import path from 'path' +import fs from 'fs' +import { exec } from 'child_process' +import { fileURLToPath } from 'url' + +const __filename = fileURLToPath(import.meta.url) +const __dirname = path.dirname(__filename) + +const runner = path.join(__dirname, '/../../bin/codecept.js') +const codecept_dir = path.join(__dirname, '/../data/sandbox/configs/last-failed') +const config = `--config ${codecept_dir}/codecept.conf.js` +const outputDir = path.join(codecept_dir, 'output') +const resultFile = path.join(outputDir, 'result.json') + +const run = (command, env = {}) => + new Promise(resolve => { + exec(`${runner} ${command} ${config}`, { env: { ...process.env, LAST_FAILED_FIXED: '', ...env } }, (err, stdout) => { + resolve({ code: err ? err.code : 0, stdout }) + }) + }) + +const reportedTests = () => { + const { tests } = JSON.parse(fs.readFileSync(resultFile, 'utf8')) + return Object.fromEntries(tests.map(test => [test.title, test.state])) +} + +const executedTests = stdout => + stdout + .split('\n') + .filter(line => line.startsWith('executed: ')) + .map(line => line.replace('executed: ', '').trim()) + .sort() + +describe('--last-failed', function () { + this.timeout(60000) + + beforeEach(() => { + fs.rmSync(outputDir, { recursive: true, force: true }) + }) + + it('should save a report with test states after a run', async () => { + const { code } = await run('run') + expect(code).toEqual(1) + expect(reportedTests()).toEqual({ + 'adds numbers': 'passed', + 'divides numbers @smoke': 'failed', + 'multiplies numbers': 'failed', + 'concats strings': 'passed', + 'uppercases strings @smoke': 'failed', + }) + }) + + it('should rerun only failed tests', async () => { + await run('run') + const { code, stdout } = await run('run --last-failed') + expect(code).toEqual(1) + expect(executedTests(stdout)).toEqual(['divides numbers', 'multiplies numbers', 'uppercases strings']) + expect(stdout).toContain('0 passed, 3 failed') + }) + + it('should drop fixed tests from the next --last-failed run', async () => { + await run('run') + const fixedRun = await run('run --last-failed', { LAST_FAILED_FIXED: '1' }) + expect(fixedRun.stdout).toContain('1 passed, 2 failed') + expect(reportedTests()).toEqual({ + 'divides numbers @smoke': 'failed', + 'multiplies numbers': 'passed', + 'uppercases strings @smoke': 'failed', + }) + + const { stdout } = await run('run --last-failed') + expect(executedTests(stdout)).toEqual(['divides numbers', 'uppercases strings']) + }) + + it('should run intersection with --grep', async () => { + await run('run') + const { stdout } = await run('run --last-failed --grep "@smoke"') + expect(executedTests(stdout)).toEqual(['divides numbers', 'uppercases strings']) + expect(stdout).toContain('0 passed, 2 failed') + }) + + it('should rerun only failed tests in workers', async () => { + const fullRun = await run('run-workers 2') + expect(fullRun.stdout).toContain('2 passed, 3 failed') + expect(reportedTests()).toMatchObject({ + 'adds numbers': 'passed', + 'divides numbers @smoke': 'failed', + 'multiplies numbers': 'failed', + 'concats strings': 'passed', + 'uppercases strings @smoke': 'failed', + }) + + const { code, stdout } = await run('run-workers 2 --last-failed') + expect(code).toEqual(1) + expect(stdout).toContain('Worker 1: 2 tests') + expect(stdout).toContain('Worker 2: 1 test') + expect(stdout).toContain('0 passed, 3 failed') + expect(stdout).not.toContain('adds numbers') + expect(stdout).not.toContain('concats strings') + }) + + it('should run intersection with --grep in workers by suite', async () => { + await run('run') + const { stdout } = await run('run-workers 2 --by suite --last-failed --grep "@smoke"') + expect(stdout).toContain('0 passed, 2 failed') + expect(stdout).not.toContain('multiplies numbers') + }) + + it('should rerun only failed tests in workers by pool', async () => { + await run('run') + const { stdout } = await run('run-workers 2 --by pool --last-failed') + expect(stdout).toContain('0 passed, 3 failed') + expect(reportedTests()).toEqual({ + 'divides numbers @smoke': 'failed', + 'multiplies numbers': 'failed', + 'uppercases strings @smoke': 'failed', + }) + }) + + it('should fail when there is no previous run', async () => { + for (const command of ['run --last-failed', 'run-workers 2 --last-failed']) { + const { code, stdout } = await run(command) + expect(code).toEqual(1) + expect(stdout).toContain('No previous run found') + expect(stdout).toContain('run tests once first') + } + }) + + it('should run nothing when the last run had no failures', async () => { + await run('run --grep "adds numbers"') + for (const command of ['run --last-failed', 'run-workers 2 --last-failed']) { + const { code, stdout } = await run(command) + expect(code).toEqual(0) + expect(stdout).toContain('No failed tests in the last run') + expect(stdout).not.toContain('executed:') + expect(stdout).not.toContain('passed') + } + }) +}) From 4f5403d578bf790a1dc171475b766fa001be89f5 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Mon, 5 Oct 2026 00:07:41 +0300 Subject: [PATCH 2/2] fix: use Object type in filterLastFailedTests JSDoc for typings Co-Authored-By: Claude Opus 5.5 --- lib/codecept.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/codecept.js b/lib/codecept.js index 19022b7d7..b44260884 100644 --- a/lib/codecept.js +++ b/lib/codecept.js @@ -268,7 +268,7 @@ class Codecept { /** * Keeps only tests that failed in the previous run when `--last-failed` is set. * - * @param {Mocha} mocha + * @param {Object} mocha Mocha instance with loaded tests */ filterLastFailedTests(mocha) { if (!this.opts.lastFailed) return