From 4d8ad3db32b991f06d99522f33998f266a9e495c Mon Sep 17 00:00:00 2001 From: Sergey Matyunin Date: Sun, 13 Sep 2026 12:30:19 +0300 Subject: [PATCH] =?UTF-8?q?test:=20=D1=80=D0=B0=D0=B7=D0=BB=D0=B8=D1=87?= =?UTF-8?q?=D0=B0=D1=82=D1=8C=20=D0=B8=D1=81=D1=85=D0=BE=D0=B4=D1=8B=20?= =?UTF-8?q?=D0=BC=D1=83=D1=82=D0=B0=D1=86=D0=B8=D0=BE=D0=BD=D0=BD=D0=BE?= =?UTF-8?q?=D0=B3=D0=BE=20=D0=B3=D0=B5=D0=B9=D1=82=D0=B0=20(#550)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue: #550 User-Visible: no --- docs/TESTING.md | 20 ++- scripts/mutation-gate-report.mjs | 29 +++- scripts/mutation-gate.mjs | 225 +++++++++++++++++++++------ scripts/mutation-guard-outcome.mjs | 155 ++++++++++++++++++ test/mutation-gate-report.test.mjs | 19 +++ test/mutation-gate.test.mjs | 38 ++++- test/mutation-guard-outcome.test.mjs | 125 +++++++++++++++ 7 files changed, 554 insertions(+), 57 deletions(-) create mode 100644 scripts/mutation-guard-outcome.mjs create mode 100644 test/mutation-guard-outcome.test.mjs diff --git a/docs/TESTING.md b/docs/TESTING.md index f3905c2f..22a94234 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -51,6 +51,17 @@ GUARD_INPUTS` (умолчание `backend-test-guard.mjs` — Бандл собирается только мутантам с браузерным гвардом; компиляция тестов в worktree стартует с тёплого `test-build/` основного дерева. +С #550 ненулевой exit сам по себе не означает «мутант пойман». Цепочка +`setup && ... && oracle` разбирается по шагам: ошибки компиляции, сборки, +загрузки/collection теста дают `setup-failure`; неприменимый патч — +`invalid-mutation`; timeout, signal и отсутствие exit status — +`infrastructure-interruption`; зелёный oracle — `survived`. Только падение +последнего заявленного oracle даёт `assertion-killed` и может попасть в +ledger. Compile-time проверка допустима как самостоятельный свидетель лишь с +явным полем определения `oracle: 'compile'`; прятать её в префиксе обычного +browser/backend guard нельзя. Все недоказанные исходы завершают гейт кодом 2 и +отдельно называются в ночном отчёте. + В CI `changed_mutants` бежит не на каждом пуше, а по запросу (#510): `workflow_dispatch validate.yml -f mutants=true` (его делают ревью-конвейер на материале ревью и слияние на кандидате), `full=true` (ночь, кнопка), PR и кандидат @@ -59,11 +70,14 @@ worktree стартует с тёплого `test-build/` основного д основном отменялись следующим пушем. Доказательство мутантов для ревью — именно dispatch-прогон на точном SHA; зелёный push-прогон им не является. -`changed_mutants` добавляет `--ledger=<файл>` — журнал пойманных -свидетелей (#481): после каждого пойманного мутанта в файл пишется отпечаток +`changed_mutants` добавляет `--ledger=<файл>` — журнал доказанных +свидетелей (#481, #550): после каждого пойманного мутанта в файл пишутся тип +доказательства (`assertion` или явно объявленный `compile`) и отпечаток его входов (файлы патча, все входы гарда по замыканию выше, объявление мутанта; строка версии продукта нормализована), и мутант с тем же отпечатком в следующем -прогоне не гоняется. +прогоне не гоняется. Схема 2 намеренно не читает старые записи без типа +доказательства: setup failure из прежнего раннера нельзя унаследовать как +зелёный результат. Журнал живёт в кэше Actions по шарду, сохраняется при любом исходе шага, так что отменённый пуш или таймаут не пропадают даром. Полный прогон и `--id` журнал не читают; `--ledger` без `--changed` — ошибка. diff --git a/scripts/mutation-gate-report.mjs b/scripts/mutation-gate-report.mjs index b782cd16..d6b6a1aa 100755 --- a/scripts/mutation-gate-report.mjs +++ b/scripts/mutation-gate-report.mjs @@ -16,6 +16,7 @@ * * FAIL : тест остался зелёным на сломанном коде → escaped * FAIL чистый прогон: красный без мутанта → redGuards + * FAIL : ошибка подготовки до заявленного теста → unverifiable * * Наивный парсер «id — это слово после FAIL» сделал бы из второй формы * мутанта по имени «чистый», которого в реестре нет, и команда `--id=чистый` @@ -34,6 +35,8 @@ export const MUTATION_EVIDENCE_SCHEMA = 'houseplan-mutation-shard-evidence/v1'; const ESCAPED_LINE = /^FAIL (\S+): тест остался зелёным на сломанном коде\s*$/; const RED_GUARD_LINE = /^FAIL чистый прогон: (.+?) красный без мутанта\s*$/; +const UNVERIFIABLE_LINE = /^FAIL (\S+): (неприменимый мутант|ошибка подготовки до заявленного теста|прерывание инфраструктуры)\s*$/; +const CLEAN_UNVERIFIABLE_LINE = /^FAIL чистая (подготовка|инфраструктура): (.+?)\s*$/; const ANY_FAIL_LINE = /^FAIL /; const FULL_SHA = /^[0-9a-f]{40}$/; @@ -152,6 +155,7 @@ export function parseShardLogs(logs, knownIds) { const known = new Set(knownIds); const escaped = new Set(); const redGuards = new Set(); + const unverifiable = []; const unparsed = []; const shards = []; for (const { shard, text } of logs) { @@ -165,6 +169,16 @@ export function parseShardLogs(logs, knownIds) { if (asEscaped && known.has(asEscaped[1])) { escaped.add(asEscaped[1]); continue; } const asRed = RED_GUARD_LINE.exec(line); if (asRed) { redGuards.add(asRed[1]); continue; } + const asUnverifiable = UNVERIFIABLE_LINE.exec(line); + if (asUnverifiable && known.has(asUnverifiable[1])) { + unverifiable.push({ shard, id: asUnverifiable[1], reason: asUnverifiable[2] }); + continue; + } + const asCleanUnverifiable = CLEAN_UNVERIFIABLE_LINE.exec(line); + if (asCleanUnverifiable) { + unverifiable.push({ shard, id: '', reason: `чистая ${asCleanUnverifiable[1]}: ${asCleanUnverifiable[2]}` }); + continue; + } unparsed.push({ shard, line }); } shards.push({ shard, status: failed ? 'failed' : 'ok' }); @@ -172,6 +186,7 @@ export function parseShardLogs(logs, knownIds) { return { escaped: [...escaped].sort(), redGuards: [...redGuards].sort(), + unverifiable, unparsed, shards: shards.sort((a, b) => a.shard - b.shard), }; @@ -193,7 +208,8 @@ export function mutationGateReport(input) { const parsed = parseShardLogs(input.logs || [], [...guards.keys()]); const evidenceErrors = [...(input.evidenceErrors || [])]; const failed = evidenceErrors.length > 0 || parsed.shards.some((s) => s.status !== 'ok') - || parsed.escaped.length > 0 || parsed.redGuards.length > 0 || parsed.unparsed.length > 0; + || parsed.escaped.length > 0 || parsed.redGuards.length > 0 + || parsed.unverifiable.length > 0 || parsed.unparsed.length > 0; const lines = []; lines.push(`Полный мутационный прогон по расписанию не прошёл: ${input.date}, \`${input.ref}\` @ \`${String(input.sha || '').slice(0, 12)}\`.`); lines.push(`Прогон: ${input.runUrl}`); @@ -232,6 +248,17 @@ export function mutationGateReport(input) { lines.push(''); for (const guard of parsed.redGuards) lines.push(`- \`${guard}\``); } + if (parsed.unverifiable.length) { + lines.push(''); + lines.push(`## Свидетели без доказательства (${parsed.unverifiable.length})`); + lines.push(''); + lines.push('Заявленный тест не дал вердикт: такой исход не записывается в ledger и не переиспользуется.'); + lines.push(''); + for (const item of parsed.unverifiable) { + const target = item.id ? `\`${item.id}\`` : `шард ${item.shard}`; + lines.push(`- ${target} — ${item.reason}`); + } + } if (parsed.unparsed.length) { lines.push(''); lines.push(`## Неразобранные строки FAIL (${parsed.unparsed.length})`); diff --git a/scripts/mutation-gate.mjs b/scripts/mutation-gate.mjs index 3a235932..03e4b796 100644 --- a/scripts/mutation-gate.mjs +++ b/scripts/mutation-gate.mjs @@ -41,6 +41,9 @@ import { createHash } from 'node:crypto'; import { mkdirSync } from 'node:fs'; import { withoutProductVersion } from './source-fingerprint.mjs'; import { closure, trackedFiles } from './check-inputs.mjs'; +import { + MUTATION_OUTCOME, MUTATION_PROOF, isProofOutcome, runGuardPhases, runMutationLifecycle, +} from './mutation-guard-outcome.mjs'; const repoRoot = fileURLToPath(new URL('..', import.meta.url)); @@ -3722,6 +3725,19 @@ const MUTANT_DEFINITIONS = [ replace: " if (asEscaped) { escaped.add(asEscaped[1]); continue; }\n if (/^FAIL (\\S+)/.test(line)) { escaped.add(line.split(' ')[1].replace(/:$/, '')); continue; }", }], }, + { + id: 'mutation-report-setup-as-unparsed', + guard: 'node --test --test-name-pattern="#550: setup/invalid/interruption" ' + + 'test/mutation-gate-report.test.mjs', + because: 'setup, invalid-patch and infrastructure outcomes must be called out as missing ' + + 'evidence; losing their parser branch buries the reason in generic text and invites a ' + + 'false escaped/caught interpretation (#550 AC4)', + patches: [{ + file: 'scripts/mutation-gate-report.mjs', + find: ' if (asUnverifiable && known.has(asUnverifiable[1])) {', + replace: ' if (false && asUnverifiable && known.has(asUnverifiable[1])) {', + }], + }, { id: 'mutation-report-accepts-foreign-material', guard: 'node --test --test-name-pattern="foreign SHA и отсутствующий шард" ' @@ -3742,14 +3758,19 @@ const MUTANT_DEFINITIONS = [ + 'skip it on every later push and hide the exact rot the gate exists for (#481)', patches: [{ file: 'scripts/mutation-gate.mjs', - find: ' if (!runMutant(entry.mutant)) ' + 'continue;\n caught++;', - replace: ' if (!runMutant(entry.mutant)) { if (ledger) recordCaught(ledgerArg, ledger, entry.mutant, entry.fingerprint); continue; }\n caught++;', + find: ' if (!isProofOutcome(outcome)) {\n' + + ' if (outcome.kind !== MUTATION_OUTCOME.SURVIVED) unverifiable = true;', + replace: ' if (!isProofOutcome(outcome)) {\n' + + " if (ledger) recordCaught(ledgerArg, ledger, entry.mutant, entry.fingerprint, 'assertion');\n" + + ' if (outcome.kind !== MUTATION_OUTCOME.SURVIVED) unverifiable = true;', }, { // Unit-наблюдаемая половина того же контракта: пустой отпечаток в // записи считался бы «совпавшим» с любым — журнал перестаёт сравнивать. file: 'scripts/mutation-gate.mjs', - find: " if (ledger.caught[mutant.id] === fingerprint) " + "skipped.push(mutant);", - replace: " if (ledger.caught[mutant.id] === fingerprint || mutant.id in ledger.caught) skipped.push(mutant);", + find: ' if (proof?.fingerprint === fingerprint && proof.proof === expectedProof\n' + + ' && validProof(proof.proof)) skipped.push(mutant);', + replace: ' if ((proof?.fingerprint === fingerprint && proof.proof === expectedProof\n' + + ' && validProof(proof.proof)) || mutant.id in ledger.caught) skipped.push(mutant);', }], }, { @@ -3903,8 +3924,36 @@ const MUTANT_DEFINITIONS = [ + 'or killed by the timeout must keep what it proved, or the snowball returns (#481)', patches: [{ file: 'scripts/mutation-gate.mjs', - find: " ledger.caught[mutant.id] = fingerprint;\n mkdirSync(dirname(file), " + "{ recursive: true });\n writeFileSync(file,", - replace: " ledger.caught[mutant.id] = fingerprint;\n mkdirSync(dirname(file), { recursive: true });\n if (Object.keys(ledger.caught).length > 1) writeFileSync(file,", + find: ' ledger.caught[mutant.id] = { fingerprint, proof };\n' + + ' mkdirSync(dirname(file), { recursive: true });\n writeFileSync(file,', + replace: ' ledger.caught[mutant.id] = { fingerprint, proof };\n' + + ' mkdirSync(dirname(file), { recursive: true });\n' + + ' if (Object.keys(ledger.caught).length > 1) writeFileSync(file,', + }], + }, + { + id: 'mutation-outcome-setup-counts-as-assertion', + guard: 'node --test test/mutation-guard-outcome.test.mjs', + because: 'a compile or preparation failure before the declared oracle must never be reusable ' + + 'proof that the named assertion executed and killed the mutant (#550 AC1-AC3)', + patches: [{ + file: 'scripts/mutation-guard-outcome.mjs', + find: " if (phase === 'setup') {", + replace: " if (false && phase === 'setup') {", + }], + }, + { + id: 'mutation-ledger-accepts-setup-proof', + guard: 'node --test --test-name-pattern="#550 fixture: clean setup|#481 AC3" ' + + 'test/mutation-guard-outcome.test.mjs test/mutation-gate.test.mjs', + because: 'ledger reuse may only carry an assertion or an explicitly declared compile-time ' + + 'witness; admitting setup failures makes a transient broken runner green forever (#550 AC4)', + patches: [{ + file: 'scripts/mutation-gate.mjs', + find: 'const validProof = (proof) => Object.values(MUTATION_PROOF)' + + '.includes(proof);', + replace: "const validProof = (proof) => proof === 'setup' || Object.values(MUTATION_PROOF)" + + '.includes(proof);', }], }, { @@ -9266,6 +9315,8 @@ export function applyPatches(root, patches) { function sh(cmd, cwd, extraEnv = {}) { return spawnSync(cmd, { cwd, shell: true, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, + timeout: Number(process.env.MUTATION_COMMAND_TIMEOUT_MS) || 180_000, + killSignal: 'SIGTERM', env: { ...process.env, ...extraEnv }, }); } @@ -9342,16 +9393,18 @@ function seedTestBuild(dir) { /** * Собрать `test-build/` из мутированного src в каталоге мутанта. * - * Код выхода `tsc` игнорируется сознательно, по той же причине, что и в - * `buildBundle`: мутант воспроизводит поломку, а не образцовый код, и имеет - * право быть нестрогим по типам. Доказательством служит появление каталога — - * если его нет, падаем громко, а не отдаём тесту пустоту. + * `tsc` здесь — подготовка, а не заявленный оракул. Его отказ не может + * считаться падением последующего теста (#550), даже если тёплый каталог от + * прошлого дерева всё ещё существует. */ function buildTestBuild(dir) { seedTestBuild(dir); - sh('npx tsc -p tsconfig.test.json', dir); + const built = sh('npx tsc -p tsconfig.test.json', dir); + if (built.status !== 0) { + throw new Error(`test-build не скомпилировался в мутанте:\n${(built.stderr || built.stdout || built.error?.message || '').slice(-2000)}`); + } const fixed = sh('node scripts/fix-test-build.mjs', dir); - if (!existsSync(join(dir, 'test-build'))) { + if (fixed.status !== 0 || !existsSync(join(dir, 'test-build'))) { throw new Error(`test-build не собрался в мутанте:\n${(fixed.stderr || fixed.stdout).slice(-2000)}`); } } @@ -9361,31 +9414,66 @@ function buildBundle(dir) { // типам — он воспроизводит поломку, а не образцовый код. const built = sh('npx rollup -c', dir); if (built.status !== 0) { - throw new Error(`сборка мутанта упала:\n${(built.stderr || built.stdout).slice(-2000)}`); + throw new Error(`сборка мутанта упала:\n${String(built.stderr || built.stdout || built.error?.message || '').slice(-2000)}`); } const synced = sh('node scripts/bundle-sync.mjs', dir); if (synced.status !== 0) { - throw new Error(`дерево бандла мутанта не синхронизировалось:\n${(synced.stderr || synced.stdout).slice(-2000)}`); + throw new Error(`дерево бандла мутанта не синхронизировалось:\n${String(synced.stderr || synced.stdout || synced.error?.message || '').slice(-2000)}`); } } +function printMutantOutcome(mutant, outcome) { + if (outcome.kind === MUTATION_OUTCOME.ASSERTION_KILLED) { + console.log(`ok ${mutant.id}: заявленный тест покраснел на мутанте`); + return; + } + if (outcome.kind === MUTATION_OUTCOME.COMPILE_KILLED) { + console.log(`ok ${mutant.id}: явный compile-time свидетель поймал мутант`); + return; + } + if (outcome.kind === MUTATION_OUTCOME.SURVIVED) { + // Формат читает mutation-gate-report.mjs — менять синхронно. + console.log(`FAIL ${mutant.id}: тест остался зелёным на сломанном коде`); + console.log(` guard: ${mutant.guard}`); + console.log(` ${mutant.because}`); + return; + } + const labels = { + [MUTATION_OUTCOME.INVALID]: 'неприменимый мутант', + [MUTATION_OUTCOME.SETUP]: 'ошибка подготовки до заявленного теста', + [MUTATION_OUTCOME.INTERRUPTED]: 'прерывание инфраструктуры', + }; + console.log(`FAIL ${mutant.id}: ${labels[outcome.kind] || outcome.kind}`); + if (outcome.command) console.log(` command: ${outcome.command}`); + if (outcome.detail) console.log(` ${outcome.detail}`); +} + function runMutant(mutant) { - const dir = makeWorktree(); + let dir; try { - applyPatches(dir, mutant.patches); - if (guardNeedsBundle(mutant.guard)) buildBundle(dir); - if (guardNeedsTestBuild(mutant.guard)) buildTestBuild(dir); - const guard = sh(mutant.guard, dir); - if (guard.status === 0) { - console.log(`FAIL ${mutant.id}: тест остался зелёным на сломанном коде`); - console.log(` guard: ${mutant.guard}`); - console.log(` ${mutant.because}`); - return false; - } - console.log(`ok ${mutant.id}: тест покраснел, как обязан`); - return true; + dir = makeWorktree(); + const outcome = runMutationLifecycle({ + apply: () => applyPatches(dir, mutant.patches), + prepare: () => { + if (guardNeedsBundle(mutant.guard)) buildBundle(dir); + if (guardNeedsTestBuild(mutant.guard)) buildTestBuild(dir); + }, + guard: mutant.guard, + proof: mutant.oracle || MUTATION_PROOF.ASSERTION, + execute: (command) => sh(command, dir), + }); + printMutantOutcome(mutant, outcome); + return outcome; + } catch (error) { + const outcome = { + kind: MUTATION_OUTCOME.INTERRUPTED, + detail: error.message, + command: '', + }; + printMutantOutcome(mutant, outcome); + return outcome; } finally { - dropWorktree(dir); + if (dir) dropWorktree(dir); } } @@ -9395,14 +9483,30 @@ function runCleanGuards(mutants) { const guards = [...new Set(mutants.map((m) => m.guard))]; const dir = makeWorktree(); try { - if (guards.some(guardNeedsBundle)) buildBundle(dir); - // Один worktree на все чистые гварды — значит и компиляция одна. - if (guards.some(guardNeedsTestBuild)) buildTestBuild(dir); + try { + if (guards.some(guardNeedsBundle)) buildBundle(dir); + // Один worktree на все чистые гварды — значит и компиляция одна. + if (guards.some(guardNeedsTestBuild)) buildTestBuild(dir); + } catch (error) { + console.log(`FAIL чистая подготовка: ${error.message}`); + return false; + } for (const guard of guards) { - const result = sh(guard, dir); - if (result.status !== 0) { - console.log(`FAIL чистый прогон: ${guard} красный без мутанта`); - console.log((result.stderr || result.stdout).slice(-1500)); + const mutant = mutants.find((item) => item.guard === guard); + const outcome = runGuardPhases(guard, { + proof: mutant?.oracle || MUTATION_PROOF.ASSERTION, + execute: (command) => sh(command, dir), + }); + if (outcome.kind !== MUTATION_OUTCOME.SURVIVED) { + if (outcome.kind === MUTATION_OUTCOME.SETUP) { + console.log(`FAIL чистая подготовка: ${outcome.command}`); + } else if (outcome.kind === MUTATION_OUTCOME.INTERRUPTED) { + console.log(`FAIL чистая инфраструктура: ${outcome.command || guard}`); + } else { + // Формат читает mutation-gate-report.mjs — менять синхронно. + console.log(`FAIL чистый прогон: ${guard} красный без мутанта`); + } + if (outcome.detail) console.log(outcome.detail.slice(-1500)); return false; } console.log(`ok чистый прогон: ${guard}`); @@ -9580,7 +9684,12 @@ export function witnessFingerprint(mutant, { normalize = withoutProductVersion(root), } = {}) { const hash = createHash('sha256'); - hash.update(JSON.stringify({ id: mutant.id, guard: mutant.guard, patches: mutant.patches })); + hash.update(JSON.stringify({ + id: mutant.id, + guard: mutant.guard, + oracle: mutant.oracle || MUTATION_PROOF.ASSERTION, + patches: mutant.patches, + })); hash.update('\0'); const text = (file) => String(read(file)).replace(/\r\n?/g, '\n'); // Сторона патча — только область якоря (#518); сторона гарда — файл целиком: @@ -9601,21 +9710,33 @@ export function witnessFingerprint(mutant, { return hash.digest('hex'); } -export const LEDGER_SCHEMA = 1; +export const LEDGER_SCHEMA = 2; -/** Журнал пойманных свидетелей: `{ schema, caught: { [id]: fingerprint } }`. */ +const emptyLedger = () => ({ schema: LEDGER_SCHEMA, caught: {} }); +const validProof = (proof) => Object.values(MUTATION_PROOF).includes(proof); + +/** + * Журнал доказанных свидетелей. + * + * Schema 2 намеренно не принимает старые строки fingerprint: до #550 они не + * доказывали, что упал именно oracle, а не tsc/setup перед ним. + */ export function readLedger(file) { - if (!file || !existsSync(file)) return { schema: LEDGER_SCHEMA, caught: {} }; + if (!file || !existsSync(file)) return emptyLedger(); try { const parsed = JSON.parse(readFileSync(file, 'utf8')); if (parsed?.schema !== LEDGER_SCHEMA || typeof parsed.caught !== 'object' || !parsed.caught) { - return { schema: LEDGER_SCHEMA, caught: {} }; + return emptyLedger(); } - return { schema: LEDGER_SCHEMA, caught: { ...parsed.caught } }; + const caught = {}; + for (const [id, value] of Object.entries(parsed.caught)) { + if (typeof value?.fingerprint === 'string' && validProof(value.proof)) caught[id] = { ...value }; + } + return { schema: LEDGER_SCHEMA, caught }; } catch { // Битый журнал — не отказ гейта: он лишь сужает работу, и пустой журнал // означает «гонять всё отобранное», то есть прежнее поведение. - return { schema: LEDGER_SCHEMA, caught: {} }; + return emptyLedger(); } } @@ -9624,8 +9745,9 @@ export function readLedger(file) { * или упавший по таймауту шард обязан сохранить уже сделанное, иначе * следующий пуш начинает с нуля (снежный ком #481). */ -export function recordCaught(file, ledger, mutant, fingerprint) { - ledger.caught[mutant.id] = fingerprint; +export function recordCaught(file, ledger, mutant, fingerprint, proof = MUTATION_PROOF.ASSERTION) { + if (!validProof(proof)) throw new Error(`нельзя записать недоказанный outcome в ledger: ${proof}`); + ledger.caught[mutant.id] = { fingerprint, proof }; mkdirSync(dirname(file), { recursive: true }); writeFileSync(file, `${JSON.stringify({ schema: LEDGER_SCHEMA, caught: ledger.caught }, null, 2)}\n`); } @@ -9640,7 +9762,10 @@ export function splitByLedger(mutants, ledger, fingerprintOf = (m) => witnessFin const skipped = []; for (const mutant of mutants) { const fingerprint = fingerprintOf(mutant); - if (ledger.caught[mutant.id] === fingerprint) skipped.push(mutant); + const proof = ledger.caught[mutant.id]; + const expectedProof = mutant.oracle || MUTATION_PROOF.ASSERTION; + if (proof?.fingerprint === fingerprint && proof.proof === expectedProof + && validProof(proof.proof)) skipped.push(mutant); else run.push({ mutant, fingerprint }); } return { run, skipped }; @@ -9899,12 +10024,18 @@ async function main(argv) { } if (!runCleanGuards(toRun)) return 2; let caught = 0; + let unverifiable = false; for (const entry of plan) { - if (!runMutant(entry.mutant)) continue; + const outcome = runMutant(entry.mutant); + if (!isProofOutcome(outcome)) { + if (outcome.kind !== MUTATION_OUTCOME.SURVIVED) unverifiable = true; + continue; + } caught++; - if (ledger) recordCaught(ledgerArg, ledger, entry.mutant, entry.fingerprint); + if (ledger) recordCaught(ledgerArg, ledger, entry.mutant, entry.fingerprint, outcome.proof); } console.log(`\nпоймано ${caught} из ${toRun.length}`); + if (unverifiable) return 2; return caught === toRun.length ? 0 : 1; } diff --git a/scripts/mutation-guard-outcome.mjs b/scripts/mutation-guard-outcome.mjs new file mode 100644 index 00000000..43dc9a47 --- /dev/null +++ b/scripts/mutation-guard-outcome.mjs @@ -0,0 +1,155 @@ +/** + * Honest outcome taxonomy for one mutation witness (#550). + * + * A shell command such as `tsc && fix-test-build && node --test` has two + * different meanings: the prefix prepares the oracle, while only the last + * command can prove the behavioural assertion. Treating every non-zero exit + * as "caught" turns a missing dependency or a mutant that no longer compiles + * into false test evidence. This module keeps that distinction pure and + * fixture-testable; the worktree/build orchestration remains in + * mutation-gate.mjs. + */ + +export const MUTATION_OUTCOME = Object.freeze({ + INVALID: 'invalid-mutation', + SETUP: 'setup-failure', + ASSERTION_KILLED: 'assertion-killed', + COMPILE_KILLED: 'compile-killed', + SURVIVED: 'survived', + INTERRUPTED: 'infrastructure-interruption', +}); + +export const MUTATION_PROOF = Object.freeze({ + ASSERTION: 'assertion', + COMPILE: 'compile', +}); + +const outputOf = (result = {}) => `${result.stdout || ''}\n${result.stderr || ''}`.trim(); + +/** Split `&&` only when it is a shell operator, not text inside quotes. */ +export function splitAndChain(command) { + const parts = []; + let start = 0; + let quote = ''; + let escaped = false; + const text = String(command || ''); + for (let index = 0; index < text.length - 1; index++) { + const char = text[index]; + if (escaped) { escaped = false; continue; } + if (char === '\\' && quote !== "'") { escaped = true; continue; } + if (quote) { + if (char === quote) quote = ''; + continue; + } + if (char === "'" || char === '"') { quote = char; continue; } + if (char === '&' && text[index + 1] === '&') { + const part = text.slice(start, index).trim(); + if (part) parts.push(part); + start = index + 2; + index++; + } + } + const tail = text.slice(start).trim(); + if (tail) parts.push(tail); + return parts; +} + +export function guardPhases(guard, proof = MUTATION_PROOF.ASSERTION) { + if (!Object.values(MUTATION_PROOF).includes(proof)) { + throw new Error(`unknown mutation oracle: ${proof}`); + } + const commands = splitAndChain(guard); + if (!commands.length) throw new Error('empty mutation guard'); + // A compile-time witness is deliberately explicit. Its command is the + // oracle itself; silently treating a prefix compile as evidence is forbidden. + if (proof === MUTATION_PROOF.COMPILE) { + if (commands.length !== 1) throw new Error('compile witness guard must be one command'); + return { setup: [], oracle: commands[0], proof }; + } + return { setup: commands.slice(0, -1), oracle: commands.at(-1), proof }; +} + +function interruption(result = {}) { + if (result.error || result.signal || result.status == null) { + const code = result.error?.code || result.signal || 'no-exit-status'; + return { kind: MUTATION_OUTCOME.INTERRUPTED, detail: String(code), command: '' }; + } + return null; +} + +// These mean the declared oracle could not load/collect/start. Ordinary +// exceptions from product code are intentionally absent: a test that executes +// the mutated path and crashes has still exposed the regression. +const ORACLE_SETUP_FAILURE = new RegExp([ + 'command not found', 'is not recognized as an internal or external command', + 'ENOENT', 'ERR_MODULE_NOT_FOUND', 'Cannot find (?:module|package)', + 'No module named', 'ImportError while (?:loading conftest|importing test module)', + 'ERROR collecting', 'collected 0 items', 'no tests ran', +].join('|'), 'i'); + +// Test frameworks include user/fixture text in failure diagnostics. A fixture +// may literally mention ERR_MODULE_NOT_FOUND, so a proven assertion failure +// must win over a setup-looking substring quoted inside that assertion. +const ASSERTION_EVIDENCE = /(?:ERR_ASSERTION|AssertionError|^FAILED\s+\S+)/im; + +export function classifyCommandResult(result, { phase = 'oracle', proof = MUTATION_PROOF.ASSERTION } = {}) { + const stopped = interruption(result); + if (stopped) return { ...stopped, command: result?.command || '' }; + if (Number(result.status) === 0) { + return { kind: MUTATION_OUTCOME.SURVIVED, detail: '', command: result?.command || '' }; + } + const detail = outputOf(result).slice(-2000); + if (phase === 'setup') { + return { kind: MUTATION_OUTCOME.SETUP, detail, command: result?.command || '' }; + } + if (proof !== MUTATION_PROOF.COMPILE && !ASSERTION_EVIDENCE.test(detail) + && ORACLE_SETUP_FAILURE.test(detail)) { + return { kind: MUTATION_OUTCOME.SETUP, detail, command: result?.command || '' }; + } + return { + kind: proof === MUTATION_PROOF.COMPILE + ? MUTATION_OUTCOME.COMPILE_KILLED : MUTATION_OUTCOME.ASSERTION_KILLED, + proof, + detail, + command: result?.command || '', + }; +} + +/** Execute setup commands in order, then the one declared oracle. */ +export function runGuardPhases(guard, { + proof = MUTATION_PROOF.ASSERTION, + execute, +} = {}) { + if (typeof execute !== 'function') throw new Error('mutation guard executor is required'); + let phases; + try { + phases = guardPhases(guard, proof); + } catch (error) { + return { kind: MUTATION_OUTCOME.INVALID, detail: error.message, command: String(guard || '') }; + } + for (const command of phases.setup) { + const result = execute(command, 'setup') || {}; + const outcome = classifyCommandResult({ ...result, command }, { phase: 'setup', proof }); + if (outcome.kind !== MUTATION_OUTCOME.SURVIVED) return outcome; + } + const result = execute(phases.oracle, 'oracle') || {}; + return classifyCommandResult({ ...result, command: phases.oracle }, { phase: 'oracle', proof }); +} + +/** Keep patch/preparation exceptions distinct from a test verdict. */ +export function runMutationLifecycle({ apply, prepare, guard, proof, execute }) { + try { apply(); } catch (error) { + return { kind: MUTATION_OUTCOME.INVALID, detail: error.message, command: '' }; + } + try { prepare(); } catch (error) { + return { kind: MUTATION_OUTCOME.SETUP, detail: error.message, command: '' }; + } + return runGuardPhases(guard, { proof, execute }); +} + +export function isProofOutcome(outcome) { + return (outcome?.kind === MUTATION_OUTCOME.ASSERTION_KILLED + && outcome.proof === MUTATION_PROOF.ASSERTION) + || (outcome?.kind === MUTATION_OUTCOME.COMPILE_KILLED + && outcome.proof === MUTATION_PROOF.COMPILE); +} diff --git a/test/mutation-gate-report.test.mjs b/test/mutation-gate-report.test.mjs index ae5c5871..79abce00 100644 --- a/test/mutation-gate-report.test.mjs +++ b/test/mutation-gate-report.test.mjs @@ -89,6 +89,25 @@ test('красный гард без мутанта — не сбежавший assert.equal(report.failed, true); }); +test('#550: setup/invalid/interruption reported as unverified, never escaped or caught', () => { + const report = mutationGateReport({ ...meta, guards, logs: [ + { shard: 1, text: [ + 'FAIL alpha-mutant: ошибка подготовки до заявленного теста', + 'FAIL beta-mutant: неприменимый мутант', + 'FAIL gamma-mutant: прерывание инфраструктуры', + 'FAIL чистая подготовка: npx tsc -p tsconfig.test.json', + ].join('\n') }, + ] }); + assert.deepEqual(report.escaped, []); + assert.deepEqual(report.redGuards, []); + assert.equal(report.unverifiable.length, 4); + assert.equal(report.unparsed.length, 0); + assert.match(report.body, /Свидетели без доказательства \(4\)/); + assert.match(report.body, /не записывается в ledger/); + assert.ok(!report.body.includes('--id=alpha-mutant')); + assert.equal(report.failed, true); +}); + test('id вне реестра не выдумывается, строка сохраняется как есть (#472 AC3)', () => { const report = mutationGateReport({ ...meta, guards, logs: [ { shard: 1, text: 'FAIL ghost-mutant: тест остался зелёным на сломанном коде\n' }, diff --git a/test/mutation-gate.test.mjs b/test/mutation-gate.test.mjs index 8e045e5e..a3087135 100644 --- a/test/mutation-gate.test.mjs +++ b/test/mutation-gate.test.mjs @@ -48,10 +48,17 @@ test('every guard command points at a file that exists', () => { test('every mutant explains itself', () => { const ids = new Set(); + const guardProofs = new Map(); for (const mutant of MUTANTS) { assert.ok(mutant.because && mutant.because.length > 40, `${mutant.id}: без объяснения мутант превратится в карго-культ`); assert.ok(!ids.has(mutant.id), `дубль id: ${mutant.id}`); + assert.ok(mutant.oracle == null || ['assertion', 'compile'].includes(mutant.oracle), + `${mutant.id}: неизвестный oracle ${mutant.oracle}`); + const proof = mutant.oracle || 'assertion'; + assert.ok(!guardProofs.has(mutant.guard) || guardProofs.get(mutant.guard) === proof, + `${mutant.id}: одинаковый guard объявлен с разными oracle`); + guardProofs.set(mutant.guard, proof); ids.add(mutant.id); } assert.ok(MUTANTS.length >= 6, 'стартовый набор — шесть мутантов по дырам из #85'); @@ -533,10 +540,16 @@ test('#481 AC2: по журналу пропускается только сов const a = { id: 'a', guard: 'g', patches: [] }; const b = { id: 'b', guard: 'g', patches: [] }; const c = { id: 'c', guard: 'g', patches: [] }; - const ledger = { schema: LEDGER_SCHEMA, caught: { a: 'fp-a', b: 'fp-old' } }; - const split = splitByLedger([a, b, c], ledger, (m) => `fp-${m.id}`); + const d = { id: 'd', guard: 'g', patches: [] }; + const ledger = { schema: LEDGER_SCHEMA, caught: { + a: { fingerprint: 'fp-a', proof: 'assertion' }, + b: { fingerprint: 'fp-old', proof: 'assertion' }, + d: { fingerprint: 'fp-d', proof: 'compile' }, + } }; + const split = splitByLedger([a, b, c, d], ledger, (m) => `fp-${m.id}`); assert.deepEqual(split.skipped.map((m) => m.id), ['a']); - assert.deepEqual(split.run.map((entry) => `${entry.mutant.id}:${entry.fingerprint}`), ['b:fp-b', 'c:fp-c']); + assert.deepEqual(split.run.map((entry) => `${entry.mutant.id}:${entry.fingerprint}`), + ['b:fp-b', 'c:fp-c', 'd:fp-d'], 'proof другого типа не переиспользуется'); }); test('#481 AC3: журнал пишется сразу при поимке и переживает битый/чужой файл', () => { @@ -546,14 +559,25 @@ test('#481 AC3: журнал пишется сразу при поимке и п assert.deepEqual(readLedger(file), { schema: LEDGER_SCHEMA, caught: {} }, 'нет файла — пустой журнал'); const ledger = readLedger(file); recordCaught(file, ledger, { id: 'a' }, 'fp-a'); - assert.deepEqual(JSON.parse(readFileSync(file, 'utf8')), { schema: LEDGER_SCHEMA, caught: { a: 'fp-a' } }, + assert.deepEqual(JSON.parse(readFileSync(file, 'utf8')), { schema: LEDGER_SCHEMA, caught: { + a: { fingerprint: 'fp-a', proof: 'assertion' }, + } }, 'запись появляется в файле немедленно, не в конце прогона'); - recordCaught(file, ledger, { id: 'b' }, 'fp-b'); - assert.deepEqual(readLedger(file).caught, { a: 'fp-a', b: 'fp-b' }); + recordCaught(file, ledger, { id: 'b' }, 'fp-b', 'compile'); + assert.deepEqual(readLedger(file).caught, { + a: { fingerprint: 'fp-a', proof: 'assertion' }, + b: { fingerprint: 'fp-b', proof: 'compile' }, + }); writeFileSync(file, '{ not json'); assert.deepEqual(readLedger(file).caught, {}, 'битый журнал — пустой, не отказ'); writeFileSync(file, JSON.stringify({ schema: 99, caught: { a: 'x' } })); assert.deepEqual(readLedger(file).caught, {}, 'чужая схема — пустой'); + writeFileSync(file, JSON.stringify({ schema: LEDGER_SCHEMA, caught: { + setup: { fingerprint: 'fp-setup', proof: 'setup' }, + legacy: 'fp-from-schema-1', + } })); + assert.deepEqual(readLedger(file).caught, {}, 'setup и старые строки не становятся доказательством'); + assert.throws(() => recordCaught(file, ledger, { id: 'bad' }, 'fp-bad', 'setup'), /недоказанный outcome/); } finally { rmSync(dir, { recursive: true, force: true }); } @@ -570,6 +594,8 @@ test('#481: отпечатки реестра детерминированы и const first = witnessFingerprint(MUTANTS[0]); assert.equal(witnessFingerprint(MUTANTS[0]), first); assert.notEqual(witnessFingerprint(MUTANTS[1]), first); + assert.notEqual(witnessFingerprint({ ...MUTANTS[0], oracle: 'compile' }), first, + 'смена типа доказательства инвалидирует ledger'); assert.match(first, /^[0-9a-f]{64}$/); }); diff --git a/test/mutation-guard-outcome.test.mjs b/test/mutation-guard-outcome.test.mjs new file mode 100644 index 00000000..adbd4ee1 --- /dev/null +++ b/test/mutation-guard-outcome.test.mjs @@ -0,0 +1,125 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { + MUTATION_OUTCOME, guardPhases, isProofOutcome, runGuardPhases, + runMutationLifecycle, splitAndChain, +} from '../scripts/mutation-guard-outcome.mjs'; + +const result = (status, output = '') => ({ status, stdout: output, stderr: '' }); + +test('#550: shell chain is split only on real && operators', () => { + assert.deepEqual(splitAndChain('tsc && fix && node --test test/x.test.mjs'), + ['tsc', 'fix', 'node --test test/x.test.mjs']); + assert.deepEqual(splitAndChain('node --test --test-name-pattern="a && b" test/x.test.mjs'), + ['node --test --test-name-pattern="a && b" test/x.test.mjs']); + assert.deepEqual(guardPhases('tsc && fix && node --test test/x.test.mjs'), { + setup: ['tsc', 'fix'], oracle: 'node --test test/x.test.mjs', proof: 'assertion', + }); +}); + +test('#550 fixture: clean setup failure never starts the named assertion', () => { + const called = []; + const outcome = runGuardPhases('prepare && node --test named.test.mjs', { + execute: (command) => { + called.push(command); + return result(1, 'command not found: prepare'); + }, + }); + assert.equal(outcome.kind, MUTATION_OUTCOME.SETUP); + assert.deepEqual(called, ['prepare']); + assert.equal(isProofOutcome(outcome), false); +}); + +test('#550 fixture: mutant-induced compile failure is setup, not a killed assertion', () => { + const called = []; + const outcome = runGuardPhases('npx tsc -p tsconfig.test.json && node --test named.test.mjs', { + execute: (command) => { + called.push(command); + return result(2, 'error TS2322: mutant does not type-check'); + }, + }); + assert.equal(outcome.kind, MUTATION_OUTCOME.SETUP); + assert.deepEqual(called, ['npx tsc -p tsconfig.test.json']); + assert.equal(isProofOutcome(outcome), false); +}); + +test('#550 fixture: real named assertion failure is reusable proof', () => { + const outcome = runGuardPhases('prepare && node --test named.test.mjs', { + execute: (command) => command === 'prepare' + ? result(0) : result(1, 'not ok 1 - named assertion\ncode: ERR_ASSERTION'), + }); + assert.deepEqual({ kind: outcome.kind, proof: outcome.proof }, { + kind: MUTATION_OUTCOME.ASSERTION_KILLED, proof: 'assertion', + }); + assert.equal(isProofOutcome(outcome), true); + assert.equal(isProofOutcome({ ...outcome, proof: 'setup' }), false, + 'поддельный proof не переносит killed outcome в ledger'); + const quotedFixture = runGuardPhases('node --test classifier.test.mjs', { + execute: () => result(1, + "AssertionError: expected setup-failure, got ERR_MODULE_NOT_FOUND\ncode: 'ERR_ASSERTION'"), + }); + assert.equal(quotedFixture.kind, MUTATION_OUTCOME.ASSERTION_KILLED, + 'слово из fixture внутри AssertionError не маскирует реальное падение теста'); +}); + +test('#550 fixture: green named assertion means the mutant survived', () => { + const outcome = runGuardPhases('node --test named.test.mjs', { + execute: () => result(0, 'ok 1 - named assertion'), + }); + assert.equal(outcome.kind, MUTATION_OUTCOME.SURVIVED); + assert.equal(isProofOutcome(outcome), false); +}); + +test('#550 fixture: oracle load/collection errors are setup failures', () => { + for (const output of [ + 'Error [ERR_MODULE_NOT_FOUND]: Cannot find package x', + 'ERROR collecting tests_backend/test_x.py', + 'collected 0 items', + ]) { + const outcome = runGuardPhases('node --test named.test.mjs', { + execute: () => result(1, output), + }); + assert.equal(outcome.kind, MUTATION_OUTCOME.SETUP, output); + } +}); + +test('#550 fixture: timeout and cancellation are infrastructure interruptions', () => { + const timeout = runGuardPhases('node --test named.test.mjs', { + execute: () => ({ status: null, signal: 'SIGTERM', error: { code: 'ETIMEDOUT' } }), + }); + const cancelled = runGuardPhases('node --test named.test.mjs', { + execute: () => ({ status: null, signal: 'SIGINT' }), + }); + assert.equal(timeout.kind, MUTATION_OUTCOME.INTERRUPTED); + assert.equal(timeout.detail, 'ETIMEDOUT'); + assert.equal(cancelled.kind, MUTATION_OUTCOME.INTERRUPTED); +}); + +test('#550 fixture: invalid patch and preparation exception are different outcomes', () => { + const invalid = runMutationLifecycle({ + apply: () => { throw new Error('anchor found 0 times'); }, + prepare: () => assert.fail('prepare must not run'), + guard: 'node --test named.test.mjs', + execute: () => assert.fail('oracle must not run'), + }); + const setup = runMutationLifecycle({ + apply: () => {}, + prepare: () => { throw new Error('rollup failed'); }, + guard: 'node --test named.test.mjs', + execute: () => assert.fail('oracle must not run'), + }); + assert.equal(invalid.kind, MUTATION_OUTCOME.INVALID); + assert.equal(setup.kind, MUTATION_OUTCOME.SETUP); +}); + +test('#550: compile-time proof is explicit and cannot hide in an assertion chain', () => { + const compile = runGuardPhases('npx tsc -p tsconfig.type-witness.json', { + proof: 'compile', execute: () => result(2, 'expected type error'), + }); + assert.deepEqual({ kind: compile.kind, proof: compile.proof }, { + kind: MUTATION_OUTCOME.COMPILE_KILLED, proof: 'compile', + }); + assert.equal(isProofOutcome(compile), true); + assert.throws(() => guardPhases('prepare && tsc', 'compile'), /one command/); +});