mirror of
https://github.com/Matysh/houseplan-card
synced 2026-10-01 20:29:00 +00:00
test: различать исходы мутационного гейта (#550)
Issue: #550 User-Visible: no
This commit is contained in:
@@ -16,6 +16,7 @@
|
||||
*
|
||||
* FAIL <mutant-id>: тест остался зелёным на сломанном коде → escaped
|
||||
* FAIL чистый прогон: <guard> красный без мутанта → redGuards
|
||||
* FAIL <mutant-id>: ошибка подготовки до заявленного теста → unverifiable
|
||||
*
|
||||
* Наивный парсер «id — это слово после FAIL» сделал бы из второй формы
|
||||
* мутанта по имени «чистый», которого в реестре нет, и команда `--id=чистый`
|
||||
@@ -34,6 +35,8 @@ export const MUTATION_EVIDENCE_SCHEMA = 'houseplan-mutation-shard-evidence/v1';
|
||||
|
||||
const ESCAPED_LINE = /^FAIL (\S+): тест остался зелёным на сломанном коде\s*$/;
|
||||
const RED_GUARD_LINE = /^FAIL чистый прогон: (.+?) красный без мутанта\s*$/;
|
||||
const UNVERIFIABLE_LINE = /^FAIL (\S+): (неприменимый мутант|ошибка подготовки до заявленного теста|прерывание инфраструктуры)\s*$/;
|
||||
const CLEAN_UNVERIFIABLE_LINE = /^FAIL чистая (подготовка|инфраструктура): (.+?)\s*$/;
|
||||
const ANY_FAIL_LINE = /^FAIL /;
|
||||
|
||||
const FULL_SHA = /^[0-9a-f]{40}$/;
|
||||
@@ -152,6 +155,7 @@ export function parseShardLogs(logs, knownIds) {
|
||||
const known = new Set(knownIds);
|
||||
const escaped = new Set();
|
||||
const redGuards = new Set();
|
||||
const unverifiable = [];
|
||||
const unparsed = [];
|
||||
const shards = [];
|
||||
for (const { shard, text } of logs) {
|
||||
@@ -165,6 +169,16 @@ export function parseShardLogs(logs, knownIds) {
|
||||
if (asEscaped && known.has(asEscaped[1])) { escaped.add(asEscaped[1]); continue; }
|
||||
const asRed = RED_GUARD_LINE.exec(line);
|
||||
if (asRed) { redGuards.add(asRed[1]); continue; }
|
||||
const asUnverifiable = UNVERIFIABLE_LINE.exec(line);
|
||||
if (asUnverifiable && known.has(asUnverifiable[1])) {
|
||||
unverifiable.push({ shard, id: asUnverifiable[1], reason: asUnverifiable[2] });
|
||||
continue;
|
||||
}
|
||||
const asCleanUnverifiable = CLEAN_UNVERIFIABLE_LINE.exec(line);
|
||||
if (asCleanUnverifiable) {
|
||||
unverifiable.push({ shard, id: '', reason: `чистая ${asCleanUnverifiable[1]}: ${asCleanUnverifiable[2]}` });
|
||||
continue;
|
||||
}
|
||||
unparsed.push({ shard, line });
|
||||
}
|
||||
shards.push({ shard, status: failed ? 'failed' : 'ok' });
|
||||
@@ -172,6 +186,7 @@ export function parseShardLogs(logs, knownIds) {
|
||||
return {
|
||||
escaped: [...escaped].sort(),
|
||||
redGuards: [...redGuards].sort(),
|
||||
unverifiable,
|
||||
unparsed,
|
||||
shards: shards.sort((a, b) => a.shard - b.shard),
|
||||
};
|
||||
@@ -193,7 +208,8 @@ export function mutationGateReport(input) {
|
||||
const parsed = parseShardLogs(input.logs || [], [...guards.keys()]);
|
||||
const evidenceErrors = [...(input.evidenceErrors || [])];
|
||||
const failed = evidenceErrors.length > 0 || parsed.shards.some((s) => s.status !== 'ok')
|
||||
|| parsed.escaped.length > 0 || parsed.redGuards.length > 0 || parsed.unparsed.length > 0;
|
||||
|| parsed.escaped.length > 0 || parsed.redGuards.length > 0
|
||||
|| parsed.unverifiable.length > 0 || parsed.unparsed.length > 0;
|
||||
const lines = [];
|
||||
lines.push(`Полный мутационный прогон по расписанию не прошёл: ${input.date}, \`${input.ref}\` @ \`${String(input.sha || '').slice(0, 12)}\`.`);
|
||||
lines.push(`Прогон: ${input.runUrl}`);
|
||||
@@ -232,6 +248,17 @@ export function mutationGateReport(input) {
|
||||
lines.push('');
|
||||
for (const guard of parsed.redGuards) lines.push(`- \`${guard}\``);
|
||||
}
|
||||
if (parsed.unverifiable.length) {
|
||||
lines.push('');
|
||||
lines.push(`## Свидетели без доказательства (${parsed.unverifiable.length})`);
|
||||
lines.push('');
|
||||
lines.push('Заявленный тест не дал вердикт: такой исход не записывается в ledger и не переиспользуется.');
|
||||
lines.push('');
|
||||
for (const item of parsed.unverifiable) {
|
||||
const target = item.id ? `\`${item.id}\`` : `шард ${item.shard}`;
|
||||
lines.push(`- ${target} — ${item.reason}`);
|
||||
}
|
||||
}
|
||||
if (parsed.unparsed.length) {
|
||||
lines.push('');
|
||||
lines.push(`## Неразобранные строки FAIL (${parsed.unparsed.length})`);
|
||||
|
||||
+178
-47
@@ -41,6 +41,9 @@ import { createHash } from 'node:crypto';
|
||||
import { mkdirSync } from 'node:fs';
|
||||
import { withoutProductVersion } from './source-fingerprint.mjs';
|
||||
import { closure, trackedFiles } from './check-inputs.mjs';
|
||||
import {
|
||||
MUTATION_OUTCOME, MUTATION_PROOF, isProofOutcome, runGuardPhases, runMutationLifecycle,
|
||||
} from './mutation-guard-outcome.mjs';
|
||||
|
||||
const repoRoot = fileURLToPath(new URL('..', import.meta.url));
|
||||
|
||||
@@ -3722,6 +3725,19 @@ const MUTANT_DEFINITIONS = [
|
||||
replace: " if (asEscaped) { escaped.add(asEscaped[1]); continue; }\n if (/^FAIL (\\S+)/.test(line)) { escaped.add(line.split(' ')[1].replace(/:$/, '')); continue; }",
|
||||
}],
|
||||
},
|
||||
{
|
||||
id: 'mutation-report-setup-as-unparsed',
|
||||
guard: 'node --test --test-name-pattern="#550: setup/invalid/interruption" '
|
||||
+ 'test/mutation-gate-report.test.mjs',
|
||||
because: 'setup, invalid-patch and infrastructure outcomes must be called out as missing '
|
||||
+ 'evidence; losing their parser branch buries the reason in generic text and invites a '
|
||||
+ 'false escaped/caught interpretation (#550 AC4)',
|
||||
patches: [{
|
||||
file: 'scripts/mutation-gate-report.mjs',
|
||||
find: ' if (asUnverifiable && known.has(asUnverifiable[1])) {',
|
||||
replace: ' if (false && asUnverifiable && known.has(asUnverifiable[1])) {',
|
||||
}],
|
||||
},
|
||||
{
|
||||
id: 'mutation-report-accepts-foreign-material',
|
||||
guard: 'node --test --test-name-pattern="foreign SHA и отсутствующий шард" '
|
||||
@@ -3742,14 +3758,19 @@ const MUTANT_DEFINITIONS = [
|
||||
+ 'skip it on every later push and hide the exact rot the gate exists for (#481)',
|
||||
patches: [{
|
||||
file: 'scripts/mutation-gate.mjs',
|
||||
find: ' if (!runMutant(entry.mutant)) ' + 'continue;\n caught++;',
|
||||
replace: ' if (!runMutant(entry.mutant)) { if (ledger) recordCaught(ledgerArg, ledger, entry.mutant, entry.fingerprint); continue; }\n caught++;',
|
||||
find: ' if (!isProofOutcome(outcome)) {\n'
|
||||
+ ' if (outcome.kind !== MUTATION_OUTCOME.SURVIVED) unverifiable = true;',
|
||||
replace: ' if (!isProofOutcome(outcome)) {\n'
|
||||
+ " if (ledger) recordCaught(ledgerArg, ledger, entry.mutant, entry.fingerprint, 'assertion');\n"
|
||||
+ ' if (outcome.kind !== MUTATION_OUTCOME.SURVIVED) unverifiable = true;',
|
||||
}, {
|
||||
// Unit-наблюдаемая половина того же контракта: пустой отпечаток в
|
||||
// записи считался бы «совпавшим» с любым — журнал перестаёт сравнивать.
|
||||
file: 'scripts/mutation-gate.mjs',
|
||||
find: " if (ledger.caught[mutant.id] === fingerprint) " + "skipped.push(mutant);",
|
||||
replace: " if (ledger.caught[mutant.id] === fingerprint || mutant.id in ledger.caught) skipped.push(mutant);",
|
||||
find: ' if (proof?.fingerprint === fingerprint && proof.proof === expectedProof\n'
|
||||
+ ' && validProof(proof.proof)) skipped.push(mutant);',
|
||||
replace: ' if ((proof?.fingerprint === fingerprint && proof.proof === expectedProof\n'
|
||||
+ ' && validProof(proof.proof)) || mutant.id in ledger.caught) skipped.push(mutant);',
|
||||
}],
|
||||
},
|
||||
{
|
||||
@@ -3903,8 +3924,36 @@ const MUTANT_DEFINITIONS = [
|
||||
+ 'or killed by the timeout must keep what it proved, or the snowball returns (#481)',
|
||||
patches: [{
|
||||
file: 'scripts/mutation-gate.mjs',
|
||||
find: " ledger.caught[mutant.id] = fingerprint;\n mkdirSync(dirname(file), " + "{ recursive: true });\n writeFileSync(file,",
|
||||
replace: " ledger.caught[mutant.id] = fingerprint;\n mkdirSync(dirname(file), { recursive: true });\n if (Object.keys(ledger.caught).length > 1) writeFileSync(file,",
|
||||
find: ' ledger.caught[mutant.id] = { fingerprint, proof };\n'
|
||||
+ ' mkdirSync(dirname(file), { recursive: true });\n writeFileSync(file,',
|
||||
replace: ' ledger.caught[mutant.id] = { fingerprint, proof };\n'
|
||||
+ ' mkdirSync(dirname(file), { recursive: true });\n'
|
||||
+ ' if (Object.keys(ledger.caught).length > 1) writeFileSync(file,',
|
||||
}],
|
||||
},
|
||||
{
|
||||
id: 'mutation-outcome-setup-counts-as-assertion',
|
||||
guard: 'node --test test/mutation-guard-outcome.test.mjs',
|
||||
because: 'a compile or preparation failure before the declared oracle must never be reusable '
|
||||
+ 'proof that the named assertion executed and killed the mutant (#550 AC1-AC3)',
|
||||
patches: [{
|
||||
file: 'scripts/mutation-guard-outcome.mjs',
|
||||
find: " if (phase === 'setup') {",
|
||||
replace: " if (false && phase === 'setup') {",
|
||||
}],
|
||||
},
|
||||
{
|
||||
id: 'mutation-ledger-accepts-setup-proof',
|
||||
guard: 'node --test --test-name-pattern="#550 fixture: clean setup|#481 AC3" '
|
||||
+ 'test/mutation-guard-outcome.test.mjs test/mutation-gate.test.mjs',
|
||||
because: 'ledger reuse may only carry an assertion or an explicitly declared compile-time '
|
||||
+ 'witness; admitting setup failures makes a transient broken runner green forever (#550 AC4)',
|
||||
patches: [{
|
||||
file: 'scripts/mutation-gate.mjs',
|
||||
find: 'const validProof = (proof) => Object.values(MUTATION_PROOF)'
|
||||
+ '.includes(proof);',
|
||||
replace: "const validProof = (proof) => proof === 'setup' || Object.values(MUTATION_PROOF)"
|
||||
+ '.includes(proof);',
|
||||
}],
|
||||
},
|
||||
{
|
||||
@@ -9266,6 +9315,8 @@ export function applyPatches(root, patches) {
|
||||
function sh(cmd, cwd, extraEnv = {}) {
|
||||
return spawnSync(cmd, {
|
||||
cwd, shell: true, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024,
|
||||
timeout: Number(process.env.MUTATION_COMMAND_TIMEOUT_MS) || 180_000,
|
||||
killSignal: 'SIGTERM',
|
||||
env: { ...process.env, ...extraEnv },
|
||||
});
|
||||
}
|
||||
@@ -9342,16 +9393,18 @@ function seedTestBuild(dir) {
|
||||
/**
|
||||
* Собрать `test-build/` из мутированного src в каталоге мутанта.
|
||||
*
|
||||
* Код выхода `tsc` игнорируется сознательно, по той же причине, что и в
|
||||
* `buildBundle`: мутант воспроизводит поломку, а не образцовый код, и имеет
|
||||
* право быть нестрогим по типам. Доказательством служит появление каталога —
|
||||
* если его нет, падаем громко, а не отдаём тесту пустоту.
|
||||
* `tsc` здесь — подготовка, а не заявленный оракул. Его отказ не может
|
||||
* считаться падением последующего теста (#550), даже если тёплый каталог от
|
||||
* прошлого дерева всё ещё существует.
|
||||
*/
|
||||
function buildTestBuild(dir) {
|
||||
seedTestBuild(dir);
|
||||
sh('npx tsc -p tsconfig.test.json', dir);
|
||||
const built = sh('npx tsc -p tsconfig.test.json', dir);
|
||||
if (built.status !== 0) {
|
||||
throw new Error(`test-build не скомпилировался в мутанте:\n${(built.stderr || built.stdout || built.error?.message || '').slice(-2000)}`);
|
||||
}
|
||||
const fixed = sh('node scripts/fix-test-build.mjs', dir);
|
||||
if (!existsSync(join(dir, 'test-build'))) {
|
||||
if (fixed.status !== 0 || !existsSync(join(dir, 'test-build'))) {
|
||||
throw new Error(`test-build не собрался в мутанте:\n${(fixed.stderr || fixed.stdout).slice(-2000)}`);
|
||||
}
|
||||
}
|
||||
@@ -9361,31 +9414,66 @@ function buildBundle(dir) {
|
||||
// типам — он воспроизводит поломку, а не образцовый код.
|
||||
const built = sh('npx rollup -c', dir);
|
||||
if (built.status !== 0) {
|
||||
throw new Error(`сборка мутанта упала:\n${(built.stderr || built.stdout).slice(-2000)}`);
|
||||
throw new Error(`сборка мутанта упала:\n${String(built.stderr || built.stdout || built.error?.message || '').slice(-2000)}`);
|
||||
}
|
||||
const synced = sh('node scripts/bundle-sync.mjs', dir);
|
||||
if (synced.status !== 0) {
|
||||
throw new Error(`дерево бандла мутанта не синхронизировалось:\n${(synced.stderr || synced.stdout).slice(-2000)}`);
|
||||
throw new Error(`дерево бандла мутанта не синхронизировалось:\n${String(synced.stderr || synced.stdout || synced.error?.message || '').slice(-2000)}`);
|
||||
}
|
||||
}
|
||||
|
||||
function printMutantOutcome(mutant, outcome) {
|
||||
if (outcome.kind === MUTATION_OUTCOME.ASSERTION_KILLED) {
|
||||
console.log(`ok ${mutant.id}: заявленный тест покраснел на мутанте`);
|
||||
return;
|
||||
}
|
||||
if (outcome.kind === MUTATION_OUTCOME.COMPILE_KILLED) {
|
||||
console.log(`ok ${mutant.id}: явный compile-time свидетель поймал мутант`);
|
||||
return;
|
||||
}
|
||||
if (outcome.kind === MUTATION_OUTCOME.SURVIVED) {
|
||||
// Формат читает mutation-gate-report.mjs — менять синхронно.
|
||||
console.log(`FAIL ${mutant.id}: тест остался зелёным на сломанном коде`);
|
||||
console.log(` guard: ${mutant.guard}`);
|
||||
console.log(` ${mutant.because}`);
|
||||
return;
|
||||
}
|
||||
const labels = {
|
||||
[MUTATION_OUTCOME.INVALID]: 'неприменимый мутант',
|
||||
[MUTATION_OUTCOME.SETUP]: 'ошибка подготовки до заявленного теста',
|
||||
[MUTATION_OUTCOME.INTERRUPTED]: 'прерывание инфраструктуры',
|
||||
};
|
||||
console.log(`FAIL ${mutant.id}: ${labels[outcome.kind] || outcome.kind}`);
|
||||
if (outcome.command) console.log(` command: ${outcome.command}`);
|
||||
if (outcome.detail) console.log(` ${outcome.detail}`);
|
||||
}
|
||||
|
||||
function runMutant(mutant) {
|
||||
const dir = makeWorktree();
|
||||
let dir;
|
||||
try {
|
||||
applyPatches(dir, mutant.patches);
|
||||
if (guardNeedsBundle(mutant.guard)) buildBundle(dir);
|
||||
if (guardNeedsTestBuild(mutant.guard)) buildTestBuild(dir);
|
||||
const guard = sh(mutant.guard, dir);
|
||||
if (guard.status === 0) {
|
||||
console.log(`FAIL ${mutant.id}: тест остался зелёным на сломанном коде`);
|
||||
console.log(` guard: ${mutant.guard}`);
|
||||
console.log(` ${mutant.because}`);
|
||||
return false;
|
||||
}
|
||||
console.log(`ok ${mutant.id}: тест покраснел, как обязан`);
|
||||
return true;
|
||||
dir = makeWorktree();
|
||||
const outcome = runMutationLifecycle({
|
||||
apply: () => applyPatches(dir, mutant.patches),
|
||||
prepare: () => {
|
||||
if (guardNeedsBundle(mutant.guard)) buildBundle(dir);
|
||||
if (guardNeedsTestBuild(mutant.guard)) buildTestBuild(dir);
|
||||
},
|
||||
guard: mutant.guard,
|
||||
proof: mutant.oracle || MUTATION_PROOF.ASSERTION,
|
||||
execute: (command) => sh(command, dir),
|
||||
});
|
||||
printMutantOutcome(mutant, outcome);
|
||||
return outcome;
|
||||
} catch (error) {
|
||||
const outcome = {
|
||||
kind: MUTATION_OUTCOME.INTERRUPTED,
|
||||
detail: error.message,
|
||||
command: '',
|
||||
};
|
||||
printMutantOutcome(mutant, outcome);
|
||||
return outcome;
|
||||
} finally {
|
||||
dropWorktree(dir);
|
||||
if (dir) dropWorktree(dir);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9395,14 +9483,30 @@ function runCleanGuards(mutants) {
|
||||
const guards = [...new Set(mutants.map((m) => m.guard))];
|
||||
const dir = makeWorktree();
|
||||
try {
|
||||
if (guards.some(guardNeedsBundle)) buildBundle(dir);
|
||||
// Один worktree на все чистые гварды — значит и компиляция одна.
|
||||
if (guards.some(guardNeedsTestBuild)) buildTestBuild(dir);
|
||||
try {
|
||||
if (guards.some(guardNeedsBundle)) buildBundle(dir);
|
||||
// Один worktree на все чистые гварды — значит и компиляция одна.
|
||||
if (guards.some(guardNeedsTestBuild)) buildTestBuild(dir);
|
||||
} catch (error) {
|
||||
console.log(`FAIL чистая подготовка: ${error.message}`);
|
||||
return false;
|
||||
}
|
||||
for (const guard of guards) {
|
||||
const result = sh(guard, dir);
|
||||
if (result.status !== 0) {
|
||||
console.log(`FAIL чистый прогон: ${guard} красный без мутанта`);
|
||||
console.log((result.stderr || result.stdout).slice(-1500));
|
||||
const mutant = mutants.find((item) => item.guard === guard);
|
||||
const outcome = runGuardPhases(guard, {
|
||||
proof: mutant?.oracle || MUTATION_PROOF.ASSERTION,
|
||||
execute: (command) => sh(command, dir),
|
||||
});
|
||||
if (outcome.kind !== MUTATION_OUTCOME.SURVIVED) {
|
||||
if (outcome.kind === MUTATION_OUTCOME.SETUP) {
|
||||
console.log(`FAIL чистая подготовка: ${outcome.command}`);
|
||||
} else if (outcome.kind === MUTATION_OUTCOME.INTERRUPTED) {
|
||||
console.log(`FAIL чистая инфраструктура: ${outcome.command || guard}`);
|
||||
} else {
|
||||
// Формат читает mutation-gate-report.mjs — менять синхронно.
|
||||
console.log(`FAIL чистый прогон: ${guard} красный без мутанта`);
|
||||
}
|
||||
if (outcome.detail) console.log(outcome.detail.slice(-1500));
|
||||
return false;
|
||||
}
|
||||
console.log(`ok чистый прогон: ${guard}`);
|
||||
@@ -9580,7 +9684,12 @@ export function witnessFingerprint(mutant, {
|
||||
normalize = withoutProductVersion(root),
|
||||
} = {}) {
|
||||
const hash = createHash('sha256');
|
||||
hash.update(JSON.stringify({ id: mutant.id, guard: mutant.guard, patches: mutant.patches }));
|
||||
hash.update(JSON.stringify({
|
||||
id: mutant.id,
|
||||
guard: mutant.guard,
|
||||
oracle: mutant.oracle || MUTATION_PROOF.ASSERTION,
|
||||
patches: mutant.patches,
|
||||
}));
|
||||
hash.update('\0');
|
||||
const text = (file) => String(read(file)).replace(/\r\n?/g, '\n');
|
||||
// Сторона патча — только область якоря (#518); сторона гарда — файл целиком:
|
||||
@@ -9601,21 +9710,33 @@ export function witnessFingerprint(mutant, {
|
||||
return hash.digest('hex');
|
||||
}
|
||||
|
||||
export const LEDGER_SCHEMA = 1;
|
||||
export const LEDGER_SCHEMA = 2;
|
||||
|
||||
/** Журнал пойманных свидетелей: `{ schema, caught: { [id]: fingerprint } }`. */
|
||||
const emptyLedger = () => ({ schema: LEDGER_SCHEMA, caught: {} });
|
||||
const validProof = (proof) => Object.values(MUTATION_PROOF).includes(proof);
|
||||
|
||||
/**
|
||||
* Журнал доказанных свидетелей.
|
||||
*
|
||||
* Schema 2 намеренно не принимает старые строки fingerprint: до #550 они не
|
||||
* доказывали, что упал именно oracle, а не tsc/setup перед ним.
|
||||
*/
|
||||
export function readLedger(file) {
|
||||
if (!file || !existsSync(file)) return { schema: LEDGER_SCHEMA, caught: {} };
|
||||
if (!file || !existsSync(file)) return emptyLedger();
|
||||
try {
|
||||
const parsed = JSON.parse(readFileSync(file, 'utf8'));
|
||||
if (parsed?.schema !== LEDGER_SCHEMA || typeof parsed.caught !== 'object' || !parsed.caught) {
|
||||
return { schema: LEDGER_SCHEMA, caught: {} };
|
||||
return emptyLedger();
|
||||
}
|
||||
return { schema: LEDGER_SCHEMA, caught: { ...parsed.caught } };
|
||||
const caught = {};
|
||||
for (const [id, value] of Object.entries(parsed.caught)) {
|
||||
if (typeof value?.fingerprint === 'string' && validProof(value.proof)) caught[id] = { ...value };
|
||||
}
|
||||
return { schema: LEDGER_SCHEMA, caught };
|
||||
} catch {
|
||||
// Битый журнал — не отказ гейта: он лишь сужает работу, и пустой журнал
|
||||
// означает «гонять всё отобранное», то есть прежнее поведение.
|
||||
return { schema: LEDGER_SCHEMA, caught: {} };
|
||||
return emptyLedger();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9624,8 +9745,9 @@ export function readLedger(file) {
|
||||
* или упавший по таймауту шард обязан сохранить уже сделанное, иначе
|
||||
* следующий пуш начинает с нуля (снежный ком #481).
|
||||
*/
|
||||
export function recordCaught(file, ledger, mutant, fingerprint) {
|
||||
ledger.caught[mutant.id] = fingerprint;
|
||||
export function recordCaught(file, ledger, mutant, fingerprint, proof = MUTATION_PROOF.ASSERTION) {
|
||||
if (!validProof(proof)) throw new Error(`нельзя записать недоказанный outcome в ledger: ${proof}`);
|
||||
ledger.caught[mutant.id] = { fingerprint, proof };
|
||||
mkdirSync(dirname(file), { recursive: true });
|
||||
writeFileSync(file, `${JSON.stringify({ schema: LEDGER_SCHEMA, caught: ledger.caught }, null, 2)}\n`);
|
||||
}
|
||||
@@ -9640,7 +9762,10 @@ export function splitByLedger(mutants, ledger, fingerprintOf = (m) => witnessFin
|
||||
const skipped = [];
|
||||
for (const mutant of mutants) {
|
||||
const fingerprint = fingerprintOf(mutant);
|
||||
if (ledger.caught[mutant.id] === fingerprint) skipped.push(mutant);
|
||||
const proof = ledger.caught[mutant.id];
|
||||
const expectedProof = mutant.oracle || MUTATION_PROOF.ASSERTION;
|
||||
if (proof?.fingerprint === fingerprint && proof.proof === expectedProof
|
||||
&& validProof(proof.proof)) skipped.push(mutant);
|
||||
else run.push({ mutant, fingerprint });
|
||||
}
|
||||
return { run, skipped };
|
||||
@@ -9899,12 +10024,18 @@ async function main(argv) {
|
||||
}
|
||||
if (!runCleanGuards(toRun)) return 2;
|
||||
let caught = 0;
|
||||
let unverifiable = false;
|
||||
for (const entry of plan) {
|
||||
if (!runMutant(entry.mutant)) continue;
|
||||
const outcome = runMutant(entry.mutant);
|
||||
if (!isProofOutcome(outcome)) {
|
||||
if (outcome.kind !== MUTATION_OUTCOME.SURVIVED) unverifiable = true;
|
||||
continue;
|
||||
}
|
||||
caught++;
|
||||
if (ledger) recordCaught(ledgerArg, ledger, entry.mutant, entry.fingerprint);
|
||||
if (ledger) recordCaught(ledgerArg, ledger, entry.mutant, entry.fingerprint, outcome.proof);
|
||||
}
|
||||
console.log(`\nпоймано ${caught} из ${toRun.length}`);
|
||||
if (unverifiable) return 2;
|
||||
return caught === toRun.length ? 0 : 1;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
/**
|
||||
* Honest outcome taxonomy for one mutation witness (#550).
|
||||
*
|
||||
* A shell command such as `tsc && fix-test-build && node --test` has two
|
||||
* different meanings: the prefix prepares the oracle, while only the last
|
||||
* command can prove the behavioural assertion. Treating every non-zero exit
|
||||
* as "caught" turns a missing dependency or a mutant that no longer compiles
|
||||
* into false test evidence. This module keeps that distinction pure and
|
||||
* fixture-testable; the worktree/build orchestration remains in
|
||||
* mutation-gate.mjs.
|
||||
*/
|
||||
|
||||
export const MUTATION_OUTCOME = Object.freeze({
|
||||
INVALID: 'invalid-mutation',
|
||||
SETUP: 'setup-failure',
|
||||
ASSERTION_KILLED: 'assertion-killed',
|
||||
COMPILE_KILLED: 'compile-killed',
|
||||
SURVIVED: 'survived',
|
||||
INTERRUPTED: 'infrastructure-interruption',
|
||||
});
|
||||
|
||||
export const MUTATION_PROOF = Object.freeze({
|
||||
ASSERTION: 'assertion',
|
||||
COMPILE: 'compile',
|
||||
});
|
||||
|
||||
const outputOf = (result = {}) => `${result.stdout || ''}\n${result.stderr || ''}`.trim();
|
||||
|
||||
/** Split `&&` only when it is a shell operator, not text inside quotes. */
|
||||
export function splitAndChain(command) {
|
||||
const parts = [];
|
||||
let start = 0;
|
||||
let quote = '';
|
||||
let escaped = false;
|
||||
const text = String(command || '');
|
||||
for (let index = 0; index < text.length - 1; index++) {
|
||||
const char = text[index];
|
||||
if (escaped) { escaped = false; continue; }
|
||||
if (char === '\\' && quote !== "'") { escaped = true; continue; }
|
||||
if (quote) {
|
||||
if (char === quote) quote = '';
|
||||
continue;
|
||||
}
|
||||
if (char === "'" || char === '"') { quote = char; continue; }
|
||||
if (char === '&' && text[index + 1] === '&') {
|
||||
const part = text.slice(start, index).trim();
|
||||
if (part) parts.push(part);
|
||||
start = index + 2;
|
||||
index++;
|
||||
}
|
||||
}
|
||||
const tail = text.slice(start).trim();
|
||||
if (tail) parts.push(tail);
|
||||
return parts;
|
||||
}
|
||||
|
||||
export function guardPhases(guard, proof = MUTATION_PROOF.ASSERTION) {
|
||||
if (!Object.values(MUTATION_PROOF).includes(proof)) {
|
||||
throw new Error(`unknown mutation oracle: ${proof}`);
|
||||
}
|
||||
const commands = splitAndChain(guard);
|
||||
if (!commands.length) throw new Error('empty mutation guard');
|
||||
// A compile-time witness is deliberately explicit. Its command is the
|
||||
// oracle itself; silently treating a prefix compile as evidence is forbidden.
|
||||
if (proof === MUTATION_PROOF.COMPILE) {
|
||||
if (commands.length !== 1) throw new Error('compile witness guard must be one command');
|
||||
return { setup: [], oracle: commands[0], proof };
|
||||
}
|
||||
return { setup: commands.slice(0, -1), oracle: commands.at(-1), proof };
|
||||
}
|
||||
|
||||
function interruption(result = {}) {
|
||||
if (result.error || result.signal || result.status == null) {
|
||||
const code = result.error?.code || result.signal || 'no-exit-status';
|
||||
return { kind: MUTATION_OUTCOME.INTERRUPTED, detail: String(code), command: '' };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// These mean the declared oracle could not load/collect/start. Ordinary
|
||||
// exceptions from product code are intentionally absent: a test that executes
|
||||
// the mutated path and crashes has still exposed the regression.
|
||||
const ORACLE_SETUP_FAILURE = new RegExp([
|
||||
'command not found', 'is not recognized as an internal or external command',
|
||||
'ENOENT', 'ERR_MODULE_NOT_FOUND', 'Cannot find (?:module|package)',
|
||||
'No module named', 'ImportError while (?:loading conftest|importing test module)',
|
||||
'ERROR collecting', 'collected 0 items', 'no tests ran',
|
||||
].join('|'), 'i');
|
||||
|
||||
// Test frameworks include user/fixture text in failure diagnostics. A fixture
|
||||
// may literally mention ERR_MODULE_NOT_FOUND, so a proven assertion failure
|
||||
// must win over a setup-looking substring quoted inside that assertion.
|
||||
const ASSERTION_EVIDENCE = /(?:ERR_ASSERTION|AssertionError|^FAILED\s+\S+)/im;
|
||||
|
||||
export function classifyCommandResult(result, { phase = 'oracle', proof = MUTATION_PROOF.ASSERTION } = {}) {
|
||||
const stopped = interruption(result);
|
||||
if (stopped) return { ...stopped, command: result?.command || '' };
|
||||
if (Number(result.status) === 0) {
|
||||
return { kind: MUTATION_OUTCOME.SURVIVED, detail: '', command: result?.command || '' };
|
||||
}
|
||||
const detail = outputOf(result).slice(-2000);
|
||||
if (phase === 'setup') {
|
||||
return { kind: MUTATION_OUTCOME.SETUP, detail, command: result?.command || '' };
|
||||
}
|
||||
if (proof !== MUTATION_PROOF.COMPILE && !ASSERTION_EVIDENCE.test(detail)
|
||||
&& ORACLE_SETUP_FAILURE.test(detail)) {
|
||||
return { kind: MUTATION_OUTCOME.SETUP, detail, command: result?.command || '' };
|
||||
}
|
||||
return {
|
||||
kind: proof === MUTATION_PROOF.COMPILE
|
||||
? MUTATION_OUTCOME.COMPILE_KILLED : MUTATION_OUTCOME.ASSERTION_KILLED,
|
||||
proof,
|
||||
detail,
|
||||
command: result?.command || '',
|
||||
};
|
||||
}
|
||||
|
||||
/** Execute setup commands in order, then the one declared oracle. */
|
||||
export function runGuardPhases(guard, {
|
||||
proof = MUTATION_PROOF.ASSERTION,
|
||||
execute,
|
||||
} = {}) {
|
||||
if (typeof execute !== 'function') throw new Error('mutation guard executor is required');
|
||||
let phases;
|
||||
try {
|
||||
phases = guardPhases(guard, proof);
|
||||
} catch (error) {
|
||||
return { kind: MUTATION_OUTCOME.INVALID, detail: error.message, command: String(guard || '') };
|
||||
}
|
||||
for (const command of phases.setup) {
|
||||
const result = execute(command, 'setup') || {};
|
||||
const outcome = classifyCommandResult({ ...result, command }, { phase: 'setup', proof });
|
||||
if (outcome.kind !== MUTATION_OUTCOME.SURVIVED) return outcome;
|
||||
}
|
||||
const result = execute(phases.oracle, 'oracle') || {};
|
||||
return classifyCommandResult({ ...result, command: phases.oracle }, { phase: 'oracle', proof });
|
||||
}
|
||||
|
||||
/** Keep patch/preparation exceptions distinct from a test verdict. */
|
||||
export function runMutationLifecycle({ apply, prepare, guard, proof, execute }) {
|
||||
try { apply(); } catch (error) {
|
||||
return { kind: MUTATION_OUTCOME.INVALID, detail: error.message, command: '' };
|
||||
}
|
||||
try { prepare(); } catch (error) {
|
||||
return { kind: MUTATION_OUTCOME.SETUP, detail: error.message, command: '' };
|
||||
}
|
||||
return runGuardPhases(guard, { proof, execute });
|
||||
}
|
||||
|
||||
export function isProofOutcome(outcome) {
|
||||
return (outcome?.kind === MUTATION_OUTCOME.ASSERTION_KILLED
|
||||
&& outcome.proof === MUTATION_PROOF.ASSERTION)
|
||||
|| (outcome?.kind === MUTATION_OUTCOME.COMPILE_KILLED
|
||||
&& outcome.proof === MUTATION_PROOF.COMPILE);
|
||||
}
|
||||
Reference in New Issue
Block a user