diff --git a/docs/TESTING.md b/docs/TESTING.md index e78724c6..a00157a0 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -89,7 +89,8 @@ worktree стартует с тёплого `test-build/` основного д ledger. Compile-time проверка допустима как самостоятельный свидетель лишь с явным полем определения `oracle: 'compile'`; прятать её в префиксе обычного browser/backend guard нельзя. Все недоказанные исходы завершают гейт кодом 2 и -отдельно называются в ночном отчёте. +отдельно называются в ночном отчёте. Гвард с `--test-name-pattern`, не исполнивший +ни одного теста, — тоже `setup-failure` ([#650](testing-notes/infrastructure.md#пустое-совпадение---test-name-pattern-650)). С #568 `setup-failure` в дифф-режиме **атрибутируется**: раннер прогоняет ОПРЕДЕЛЕНИЕ БАЗЫ диапазона на дереве базы и говорит, чей это отказ. diff --git a/docs/testing-notes/README.md b/docs/testing-notes/README.md index 6d39b143..809195d6 100644 --- a/docs/testing-notes/README.md +++ b/docs/testing-notes/README.md @@ -105,6 +105,7 @@ - [Lazy editor runtime and frontend asset tree (#337)](infrastructure.md#lazy-editor-runtime-and-frontend-asset-tree-337) - [Съёмка документации запускается с флагами детерминизма (#424)](infrastructure.md#съёмка-документации-запускается-с-флагами-детерминизма-424) - [Воспроизводимость съёмки документации (#410, #422)](infrastructure.md#воспроизводимость-съёмки-документации-410-422) +- [Пустое совпадение `--test-name-pattern` (#650)](infrastructure.md#пустое-совпадение---test-name-pattern-650) ## [История прогонов и партий](history.md) diff --git a/docs/testing-notes/infrastructure.md b/docs/testing-notes/infrastructure.md index a7f3b04c..532482c6 100644 --- a/docs/testing-notes/infrastructure.md +++ b/docs/testing-notes/infrastructure.md @@ -104,3 +104,21 @@ has no separate defect. Целочисленность обрезки — единственная часть съёмки, которую можно проверить без браузера: `node --test test/capture-clip.test.mjs`. + +## Пустое совпадение `--test-name-pattern` (#650) + +Зелёный `node --test` с `--test-name-pattern`, который +не исполнил ни одного теста: TAP называет только сам файл +(`ok 1 - test/x.test.mjs`), ассертов не было ни в чистом прогоне, ни на +мутанте. Так бывает, когда задача переименовала тест, а гвард реестра остался +со старым именем; раньше чистый прогон считал такого свидетеля здоровым, а +мутант — выжившим. Шаблон — это RegExp: `(`, `+`, `?` в имени экранируются +(в команде гварда `\(отбор\)`, в JS-строке реестра `\\(отбор\\)`). `mutation-gate --check` проверяет то же статически: шаблон обязан +совпасть хотя бы с одним литеральным именем `test(`/`it(`/`describe(` в файлах +гварда; если в файле есть имена `${…}`, несовпадение — предупреждение `WARN`, +иначе `FAIL`. + +Это `setup-failure` (#550): в дифф-режиме он атрибутируется как любой другой (#568) — +переименование теста в задаче даёт `introduced`, старый пустой шаблон — `pre-existing`. +Свидетель — `test/mutation-guard-outcome.test.mjs` (тесты `#650 …`); проверка +установленного `node` на настоящем выводе TAP — там же. diff --git a/scripts/mutation-gate.mjs b/scripts/mutation-gate.mjs index 435b6df6..5927fdb3 100644 --- a/scripts/mutation-gate.mjs +++ b/scripts/mutation-gate.mjs @@ -24,7 +24,9 @@ import { } from './mutation-evidence.mjs'; import { attributeSetupFailure } from './mutation-attribution.mjs'; import { guardEnvironment, planEnvironment, planEnvironmentLines } from './mutation-environment.mjs'; -import { MUTATION_OUTCOME, isProofOutcome } from './mutation-guard-outcome.mjs'; +import { + MUTATION_OUTCOME, isProofOutcome, staticTestSelectionProblems, +} from './mutation-guard-outcome.mjs'; const repoRoot = fileURLToPath(new URL('..', import.meta.url)); @@ -151,6 +153,11 @@ export async function main(argv) { if (argv.includes('--check')) { let stale = 0; + let warned = 0; + const readTest = (file) => { + const path = join(repoRoot, file); + return existsSync(path) ? readFileSync(path, 'utf8') : null; + }; for (const m of selected) { try { // #499: бандл мутанта собирает раннер (`buildBundle`, только rollup + @@ -165,12 +172,21 @@ export async function main(argv) { const hits = source.split(patch.find).length - 1; if (hits !== 1) throw new Error(`якорь найден ${hits} раз(а)`); } + // #650: a name filter that matches nothing makes the witness vacuous. + const selection = staticTestSelectionProblems(m.guard, readTest); + const error = selection.find((problem) => problem.level === 'error'); + if (error) throw new Error(error.text); + for (const problem of selection) { + console.log(`WARN ${m.id}: ${problem.text}`); + warned++; + } console.log(`ok ${m.id}`); } catch (error) { console.log(`FAIL ${m.id}: ${error.message}`); stale++; } } + if (warned) console.log(`предупреждений о шаблонах имён: ${warned}`); reportPlanMetrics(); return stale ? 2 : 0; } diff --git a/scripts/mutation-guard-outcome.mjs b/scripts/mutation-guard-outcome.mjs index 1785b1fc..a2b02609 100644 --- a/scripts/mutation-guard-outcome.mjs +++ b/scripts/mutation-guard-outcome.mjs @@ -113,10 +113,177 @@ const ORACLE_SETUP_FAILURE = new RegExp([ // must win over a setup-looking substring quoted inside that assertion. const ASSERTION_EVIDENCE = /(?:ERR_ASSERTION|AssertionError|^FAILED\s+\S+|^# Subtest:|^✖\s+)/im; +/** + * A `--test-name-pattern` that matches no test (#650). + * + * `node --test --test-name-pattern=X file` with no test named like X exits 0: + * TAP reports only the file itself (`ok 1 - test/file.test.mjs`). On the clean + * run that reads as a healthy witness and on the mutant as `survived`, although + * no assertion executed either time — typically a task renamed the test. It is + * a setup failure of the oracle, not a verdict about the code. + */ +const TEST_FILE = /\.(?:m|c)?[jt]s$/; +const looksLikePath = (name) => /^[\w@.~/\\:-]+$/.test(name) && TEST_FILE.test(name); + +/** Shell-like words; quotes group and are removed, `\` escapes outside '...'. */ +export function shellWords(command) { + const words = []; + let word = ''; + let quote = ''; + let active = false; + const text = String(command || ''); + for (let index = 0; index < text.length; index++) { + const char = text[index]; + if (quote) { + if (char === quote) { quote = ''; continue; } + // POSIX: inside "…" a backslash escapes only $ ` " \ and newline. + if (char === '\\' && quote === '"' && /[$`"\\\n]/.test(text[index + 1] || '')) { + word += text[++index]; + continue; + } + word += char; + continue; + } + if (char === "'" || char === '"') { quote = char; active = true; continue; } + if (char === '\\' && index + 1 < text.length) { word += text[++index]; active = true; continue; } + if (/\s/.test(char)) { + if (active) words.push(word); + word = ''; active = false; + continue; + } + word += char; active = true; + } + if (active) words.push(word); + return words; +} + +/** + * `{ patterns, files }` of one `node --test` command that filters by name, or + * `null` when the command runs no name filter (nothing to prove empty). + */ +export function nodeTestSelection(command) { + const words = shellWords(command); + const node = words.findIndex((word, index) => /(?:^|\/)node(?:\.exe)?$/.test(word) + && words[index + 1] === '--test'); + if (node < 0) return null; + const patterns = []; + const files = []; + for (let index = node + 2; index < words.length; index++) { + const word = words[index]; + if (word.startsWith('--test-name-pattern=')) { patterns.push(word.slice('--test-name-pattern='.length)); continue; } + if (word === '--test-name-pattern' && index + 1 < words.length) { patterns.push(words[++index]); continue; } + if (word.startsWith('-')) continue; + if (TEST_FILE.test(word)) files.push(word); + } + return patterns.length ? { patterns, files } : null; +} + +/** Node's pattern grammar: `/source/flags` or a plain RegExp source. */ +export function testNamePatternRegExp(pattern) { + const literal = /^\/(.*)\/([a-z]*)$/s.exec(pattern); + return literal ? new RegExp(literal[1], literal[2]) : new RegExp(pattern); +} + +/** + * Names of tests the reporter says were executed (TAP and spec), excluding + * the per-file wrapper and tests skipped by the pattern. `null` when the + * output carries no reporter summary at all — then nothing can be judged. + */ +export function executedTestNames(output) { + const text = String(output || ''); + if (!/^\s*(?:#|ℹ) tests \d+/m.test(text)) return null; + const names = []; + for (const match of text.matchAll(/^\s*(?:not )?ok \d+ - (.+?)\s*$/gm)) { + const [name, directive = ''] = match[1].split(/\s+#\s+/); + if (/^SKIP\b/i.test(directive)) continue; + const plain = name.replace(/\\#/g, '#').trim(); + if (!looksLikePath(plain)) names.push(plain); + } + for (const match of text.matchAll(/^\s*[✔✖] (.+?) \(\d[\d.]*m?s\)\s*$/gm)) { + const plain = match[1].trim(); + if (!looksLikePath(plain) && !names.includes(plain)) names.push(plain); + } + return names; +} + +/** Diagnostic text when a name-filtered oracle executed no test; else `null`. */ +export function emptyTestSelection(command, output) { + const selection = nodeTestSelection(command); + if (!selection) return null; + const names = executedTestNames(output); + if (names === null || names.length) return null; + return `--test-name-pattern ${selection.patterns.map((p) => JSON.stringify(p)).join(', ')}` + + ` не совпал ни с одним тестом в ${selection.files.join(' ') || 'выбранных файлах'}` + + ' — свидетель не исполнил ни одного ассерта (#650)'; +} + +/** + * Static test names of a test source: `test(`, `it(`, `describe(`, `suite(` + * and `t.test(` with a literal first argument. A template literal with `${…}` + * is dynamic: it cannot be proven absent. + */ +export function staticTestNames(source) { + const names = []; + let dynamic = false; + const call = /(?:^|[^\w.$]|\bt\.)(?:test|it|describe|suite)(?:\.(?:only|skip|todo))?\(\s*(?:(['"])((?:\\.|(?!\1)[^\\\n])*)\1|`((?:\\.|[^\\`])*)`)/g; + for (const match of String(source || '').matchAll(call)) { + if (match[3] !== undefined) { + if (/\$\{/.test(match[3])) { dynamic = true; continue; } + names.push(match[3]); + } else { + names.push(match[2].replace(/\\(.)/g, '$1')); + } + } + return { names, dynamic }; +} + +/** + * Registry-time check of one guard: every name-filtered `node --test` must + * match at least one static test name in its files. `read(file)` returns the + * source or `null` for a missing file. + */ +export function staticTestSelectionProblems(guard, read) { + const problems = []; + for (const command of splitAndChain(guard)) { + const selection = nodeTestSelection(command); + if (!selection) continue; + if (!selection.files.length || selection.files.some((file) => /[*?[]/.test(file))) continue; + let regexps; + try { + regexps = selection.patterns.map(testNamePatternRegExp); + } catch (error) { + problems.push({ level: 'error', text: `--test-name-pattern не RegExp: ${error.message}` }); + continue; + } + const names = []; + let dynamic = false; + const missing = selection.files.filter((file) => read(file) == null); + if (missing.length) { + problems.push({ level: 'error', text: `нет файла ${missing.join(' ')}` }); + continue; + } + for (const file of selection.files) { + const found = staticTestNames(read(file)); + names.push(...found.names); + dynamic ||= found.dynamic; + } + if (names.some((name) => regexps.some((regexp) => regexp.test(name)))) continue; + const text = `--test-name-pattern ${selection.patterns.map((p) => JSON.stringify(p)).join(', ')}` + + ` не совпадает ни с одним тестом в ${selection.files.join(' ')} (#650)`; + problems.push(dynamic + ? { level: 'warn', text: `${text}; в файле есть имена с \${…} — проверить нельзя` } + : { level: 'error', text }); + } + return problems; +} + export function classifyCommandResult(result, { phase = 'oracle', proof = MUTATION_PROOF.ASSERTION } = {}) { const stopped = interruption(result); if (stopped) return { ...stopped, command: result?.command || '' }; if (Number(result.status) === 0) { + // #650: a green oracle that executed no named test proves nothing either way. + const empty = phase === 'oracle' ? emptyTestSelection(result?.command, outputOf(result)) : null; + if (empty) return { kind: MUTATION_OUTCOME.SETUP, detail: empty, command: result?.command || '' }; return { kind: MUTATION_OUTCOME.SURVIVED, detail: '', command: result?.command || '' }; } const detail = outputOf(result).slice(-2000); diff --git a/scripts/mutation-registry.mjs b/scripts/mutation-registry.mjs index 1a1c454a..760bbf3f 100644 --- a/scripts/mutation-registry.mjs +++ b/scripts/mutation-registry.mjs @@ -4753,6 +4753,27 @@ const MUTANT_DEFINITIONS = [ replace: ' if (!hunks || !hunks.length) return false;', }], }, + // #650: a --test-name-pattern that matches nothing is a vacuous witness. + { + id: 'empty-test-selection-reads-as-survived', + guard: "node --test --test-name-pattern=\"#650 (clean run|mutant run|the installed node)\" test/mutation-guard-outcome.test.mjs", + because: '#650: a green oracle that executed no named test proves nothing; calling it ' + + 'survived blames the code, and a clean run calls it a healthy witness', + patches: [{ file: 'scripts/mutation-guard-outcome.mjs', find: " const empty = phase === 'oracle' ? emptyTestSelection(result?.command, outputOf(result)) : null;", replace: " const empty = null;" }], + }, + { + id: 'static-test-selection-accepts-unmatched', + guard: "node --test --test-name-pattern=\"#650 static check\" test/mutation-guard-outcome.test.mjs", + because: '#650: mutation-gate --check must name an unmatched name filter before CI spends a shard on it', + patches: [{ file: 'scripts/mutation-guard-outcome.mjs', find: " if (names.some((name) => regexps.some((regexp) => regexp.test(name)))) continue;", replace: " if (names.length || !names.length) continue;" }], + }, + { + id: 'shell-words-drops-regex-escape', + guard: "node --test --test-name-pattern=\"#650 (selection|static check)\" test/mutation-guard-outcome.test.mjs", + because: '#650: inside double quotes the shell keeps a backslash before ( — dropping it turns an ' + + 'escaped pattern into a group and flags healthy guards as empty', + patches: [{ file: 'scripts/mutation-guard-outcome.mjs', find: " if (char === '\\\\' && quote === '\"' && /[$`\"\\\\\\n]/.test(text[index + 1] || '')) {", replace: " if (char === '\\\\' && quote === '\"' && index + 1 < text.length) {" }], + }, { id: 'ledger-version-sensitive', guard: 'node --test --test-name-pattern="#481 AC1" test/mutation-gate.test.mjs', diff --git a/test/mutation-guard-outcome.test.mjs b/test/mutation-guard-outcome.test.mjs index 3202af52..df269b6f 100644 --- a/test/mutation-guard-outcome.test.mjs +++ b/test/mutation-guard-outcome.test.mjs @@ -222,3 +222,98 @@ test('#568: раннер не красит гейт предсуществующ assert.match(source, /pre-existing-setup-failures=/, 'список назван машиночитаемой строкой: его читает человек и CI'); }); + +// ---- #650: a name filter that matches no test is a setup failure ---------- +import { mkdtempSync, rmSync, writeFileSync as writeFixture } from 'node:fs'; +import { spawnSync } from 'node:child_process'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { + emptyTestSelection, executedTestNames, nodeTestSelection, shellWords, + staticTestNames, staticTestSelectionProblems, +} from '../scripts/mutation-guard-outcome.mjs'; +import { MUTANTS } from '../scripts/mutation-registry.mjs'; + +const EMPTY_TAP = 'TAP version 13\n# Subtest: test/x.test.mjs\nok 1 - test/x.test.mjs\n ---\n' + + ' duration_ms: 40\n ...\n1..1\n# tests 1\n# suites 0\n# pass 1\n# fail 0\n'; +const NAMED_TAP = 'TAP version 13\n# Subtest: \\#650 named\nok 1 - \\#650 named\n ---\n ...\n' + + '1..1\n# tests 1\n# pass 1\n# fail 0\n'; + +test('#650 selection: patterns and files of a name-filtered node --test, POSIX quoting', () => { + assert.deepEqual(nodeTestSelection('node --test --test-name-pattern="#518 AC1/AC2 \\(отбор\\)" test/a.test.mjs'), + { patterns: ['#518 AC1/AC2 \\(отбор\\)'], files: ['test/a.test.mjs'] }, + 'inside "…" a backslash before ( stays: it is part of the RegExp'); + assert.deepEqual(nodeTestSelection("node --test --test-name-pattern 'a|b' --test-name-pattern=c x.test.mjs y.test.mjs"), + { patterns: ['a|b', 'c'], files: ['x.test.mjs', 'y.test.mjs'] }); + assert.equal(nodeTestSelection('node --test test/a.test.mjs'), null, 'no filter — nothing to prove empty'); + assert.equal(nodeTestSelection('node demo/smoke_x.mjs'), null); + assert.deepEqual(shellWords('a "b \\" c" d\\ e'), ['a', 'b " c', 'd e']); +}); + +test('#650 clean run: zero executed tests under a name filter is setup, not a healthy witness', () => { + const guard = 'node --test --test-name-pattern="renamed test" test/x.test.mjs'; + const outcome = runGuardPhases(guard, { execute: () => result(0, EMPTY_TAP) }); + assert.equal(outcome.kind, MUTATION_OUTCOME.SETUP); + assert.match(outcome.detail, /не совпал ни с одним тестом в test\/x\.test\.mjs/); + assert.equal(isProofOutcome(outcome), false); + assert.equal(runGuardPhases(guard, { execute: () => result(0, NAMED_TAP) }).kind, + MUTATION_OUTCOME.SURVIVED, 'a matched green test is an honest survivor'); + assert.equal(runGuardPhases('node --test test/x.test.mjs', { execute: () => result(0, EMPTY_TAP) }).kind, + MUTATION_OUTCOME.SURVIVED, 'without a filter an empty file is not this rule'); + assert.equal(emptyTestSelection(guard, 'no reporter output'), null, 'no summary — nothing to judge'); + assert.deepEqual(executedTestNames(NAMED_TAP), ['#650 named']); + assert.deepEqual(executedTestNames('ok 1 - t # SKIP test name does not match\n# tests 1\n'), []); +}); + +test('#650 mutant run: an empty selection on the head, healthy on the base, is introduced by the diff', () => { + const guard = 'node --test --test-name-pattern="renamed test" test/x.test.mjs'; + const head = runGuardPhases(guard, { execute: () => result(0, EMPTY_TAP) }); + const base = runGuardPhases(guard, { execute: () => result(1, 'not ok 1 - renamed test\ncode: ERR_ASSERTION') }); + assert.equal(head.kind, MUTATION_OUTCOME.SETUP); + assert.equal(setupFailureOwner(head, base), 'introduced'); + assert.equal(setupFailureOwner(head, runGuardPhases(guard, { execute: () => result(0, EMPTY_TAP) })), + 'pre-existing'); +}); + +test('#650 the installed node really reports an unmatched filter as the file alone', () => { + const dir = mkdtempSync(join(tmpdir(), 'hp-650-')); + try { + const file = join(dir, 'x.test.mjs'); + writeFixture(file, "import test from 'node:test';\ntest('real name', () => {});\n"); + const run = (pattern) => { + const command = `node --test --test-name-pattern=${JSON.stringify(pattern)} ${JSON.stringify(file)}`; + // Outside a test runner, as the gate runs guards: NODE_TEST_CONTEXT would + // switch the child to the parent's serialized protocol. + const { NODE_TEST_CONTEXT: _context, ...env } = process.env; + const r = spawnSync(command, { shell: true, encoding: 'utf8', env }); + return runGuardPhases(command, { execute: () => r }); + }; + assert.equal(run('no such name').kind, MUTATION_OUTCOME.SETUP); + assert.equal(run('real name').kind, MUTATION_OUTCOME.SURVIVED); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('#650 static check: names, regex escapes, dynamic names only warn', () => { + const source = "test('#518 AC1/AC2 (отбор): x', () => {});\nt.test(\"inner\", () => {});\n" + + 'test(`dyn ${n}`, () => {});\n'; + assert.deepEqual(staticTestNames(source), { names: ['#518 AC1/AC2 (отбор): x', 'inner'], dynamic: true }); + const read = (file) => (file === 'a.test.mjs' ? source : file === 'b.test.mjs' ? "test('b', () => {});" : null); + assert.deepEqual(staticTestSelectionProblems('node --test --test-name-pattern="AC2 \\(отбор\\)" a.test.mjs', read), []); + assert.deepEqual(staticTestSelectionProblems('tsc && node --test --test-name-pattern="nope" b.test.mjs', read) + .map((p) => p.level), ['error']); + assert.deepEqual(staticTestSelectionProblems('node --test --test-name-pattern="nope" a.test.mjs', read) + .map((p) => p.level), ['warn'], 'a ${…} name could match at run time'); + assert.deepEqual(staticTestSelectionProblems('node --test --test-name-pattern="x" gone.test.mjs', read) + .map((p) => p.level), ['error']); +}); + +test('#650 registry: every name-filtered guard matches a static test name', () => { + const read = (file) => { + try { return readFileSync(new URL(`../${file}`, import.meta.url), 'utf8'); } catch { return null; } + }; + const errors = MUTANTS.flatMap((m) => staticTestSelectionProblems(m.guard, read) + .filter((p) => p.level === 'error').map((p) => `${m.id}: ${p.text}`)); + assert.deepEqual(errors, []); +});