mirror of
https://github.com/Matysh/houseplan-card
synced 2026-09-29 03:09:36 +00:00
ci(mutants): an unmatched --test-name-pattern is setup-failure, not survived (#650)
A name-filtered `node --test` that matches no test exits 0 and TAP reports only the file itself. The clean run took that for a healthy witness and the mutant run for `survived`, although no assertion ran either time — as in Validate 36151140394 on #649, where a renamed test left a guard pattern empty. - mutation-guard-outcome: nodeTestSelection/executedTestNames/ emptyTestSelection; a green oracle with a name filter and zero executed named tests is `setup-failure` (clean run: «FAIL чистая подготовка», diff mode: attributed like any setup failure, #568). - mutation-gate --check: staticTestSelectionProblems — every pattern must match a literal test(/it(/describe( name in the guard's files; files with ${…} names only warn. POSIX quoting keeps \( inside "…". - Registry today: 349 unique name-filtered commands run on dev, all execute ≥ 1 test; --check: 0 errors, 3 warnings (dynamic names). - Tests: 6 new in test/mutation-guard-outcome.test.mjs (incl. the installed node's real TAP and a registry-wide static check); 3 mutants. - Docs: TESTING.md pointer, testing-notes/infrastructure.md section. Issue: #650 User-Visible: no
This commit is contained in:
Claude
@@ -113,10 +113,177 @@ const ORACLE_SETUP_FAILURE = new RegExp([
|
||||
// must win over a setup-looking substring quoted inside that assertion.
|
||||
const ASSERTION_EVIDENCE = /(?:ERR_ASSERTION|AssertionError|^FAILED\s+\S+|^# Subtest:|^✖\s+)/im;
|
||||
|
||||
/**
|
||||
* A `--test-name-pattern` that matches no test (#650).
|
||||
*
|
||||
* `node --test --test-name-pattern=X file` with no test named like X exits 0:
|
||||
* TAP reports only the file itself (`ok 1 - test/file.test.mjs`). On the clean
|
||||
* run that reads as a healthy witness and on the mutant as `survived`, although
|
||||
* no assertion executed either time — typically a task renamed the test. It is
|
||||
* a setup failure of the oracle, not a verdict about the code.
|
||||
*/
|
||||
const TEST_FILE = /\.(?:m|c)?[jt]s$/;
|
||||
const looksLikePath = (name) => /^[\w@.~/\\:-]+$/.test(name) && TEST_FILE.test(name);
|
||||
|
||||
/** Shell-like words; quotes group and are removed, `\` escapes outside '...'. */
|
||||
export function shellWords(command) {
|
||||
const words = [];
|
||||
let word = '';
|
||||
let quote = '';
|
||||
let active = false;
|
||||
const text = String(command || '');
|
||||
for (let index = 0; index < text.length; index++) {
|
||||
const char = text[index];
|
||||
if (quote) {
|
||||
if (char === quote) { quote = ''; continue; }
|
||||
// POSIX: inside "…" a backslash escapes only $ ` " \ and newline.
|
||||
if (char === '\\' && quote === '"' && /[$`"\\\n]/.test(text[index + 1] || '')) {
|
||||
word += text[++index];
|
||||
continue;
|
||||
}
|
||||
word += char;
|
||||
continue;
|
||||
}
|
||||
if (char === "'" || char === '"') { quote = char; active = true; continue; }
|
||||
if (char === '\\' && index + 1 < text.length) { word += text[++index]; active = true; continue; }
|
||||
if (/\s/.test(char)) {
|
||||
if (active) words.push(word);
|
||||
word = ''; active = false;
|
||||
continue;
|
||||
}
|
||||
word += char; active = true;
|
||||
}
|
||||
if (active) words.push(word);
|
||||
return words;
|
||||
}
|
||||
|
||||
/**
|
||||
* `{ patterns, files }` of one `node --test` command that filters by name, or
|
||||
* `null` when the command runs no name filter (nothing to prove empty).
|
||||
*/
|
||||
export function nodeTestSelection(command) {
|
||||
const words = shellWords(command);
|
||||
const node = words.findIndex((word, index) => /(?:^|\/)node(?:\.exe)?$/.test(word)
|
||||
&& words[index + 1] === '--test');
|
||||
if (node < 0) return null;
|
||||
const patterns = [];
|
||||
const files = [];
|
||||
for (let index = node + 2; index < words.length; index++) {
|
||||
const word = words[index];
|
||||
if (word.startsWith('--test-name-pattern=')) { patterns.push(word.slice('--test-name-pattern='.length)); continue; }
|
||||
if (word === '--test-name-pattern' && index + 1 < words.length) { patterns.push(words[++index]); continue; }
|
||||
if (word.startsWith('-')) continue;
|
||||
if (TEST_FILE.test(word)) files.push(word);
|
||||
}
|
||||
return patterns.length ? { patterns, files } : null;
|
||||
}
|
||||
|
||||
/** Node's pattern grammar: `/source/flags` or a plain RegExp source. */
|
||||
export function testNamePatternRegExp(pattern) {
|
||||
const literal = /^\/(.*)\/([a-z]*)$/s.exec(pattern);
|
||||
return literal ? new RegExp(literal[1], literal[2]) : new RegExp(pattern);
|
||||
}
|
||||
|
||||
/**
|
||||
* Names of tests the reporter says were executed (TAP and spec), excluding
|
||||
* the per-file wrapper and tests skipped by the pattern. `null` when the
|
||||
* output carries no reporter summary at all — then nothing can be judged.
|
||||
*/
|
||||
export function executedTestNames(output) {
|
||||
const text = String(output || '');
|
||||
if (!/^\s*(?:#|ℹ) tests \d+/m.test(text)) return null;
|
||||
const names = [];
|
||||
for (const match of text.matchAll(/^\s*(?:not )?ok \d+ - (.+?)\s*$/gm)) {
|
||||
const [name, directive = ''] = match[1].split(/\s+#\s+/);
|
||||
if (/^SKIP\b/i.test(directive)) continue;
|
||||
const plain = name.replace(/\\#/g, '#').trim();
|
||||
if (!looksLikePath(plain)) names.push(plain);
|
||||
}
|
||||
for (const match of text.matchAll(/^\s*[✔✖] (.+?) \(\d[\d.]*m?s\)\s*$/gm)) {
|
||||
const plain = match[1].trim();
|
||||
if (!looksLikePath(plain) && !names.includes(plain)) names.push(plain);
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
/** Diagnostic text when a name-filtered oracle executed no test; else `null`. */
|
||||
export function emptyTestSelection(command, output) {
|
||||
const selection = nodeTestSelection(command);
|
||||
if (!selection) return null;
|
||||
const names = executedTestNames(output);
|
||||
if (names === null || names.length) return null;
|
||||
return `--test-name-pattern ${selection.patterns.map((p) => JSON.stringify(p)).join(', ')}`
|
||||
+ ` не совпал ни с одним тестом в ${selection.files.join(' ') || 'выбранных файлах'}`
|
||||
+ ' — свидетель не исполнил ни одного ассерта (#650)';
|
||||
}
|
||||
|
||||
/**
|
||||
* Static test names of a test source: `test(`, `it(`, `describe(`, `suite(`
|
||||
* and `t.test(` with a literal first argument. A template literal with `${…}`
|
||||
* is dynamic: it cannot be proven absent.
|
||||
*/
|
||||
export function staticTestNames(source) {
|
||||
const names = [];
|
||||
let dynamic = false;
|
||||
const call = /(?:^|[^\w.$]|\bt\.)(?:test|it|describe|suite)(?:\.(?:only|skip|todo))?\(\s*(?:(['"])((?:\\.|(?!\1)[^\\\n])*)\1|`((?:\\.|[^\\`])*)`)/g;
|
||||
for (const match of String(source || '').matchAll(call)) {
|
||||
if (match[3] !== undefined) {
|
||||
if (/\$\{/.test(match[3])) { dynamic = true; continue; }
|
||||
names.push(match[3]);
|
||||
} else {
|
||||
names.push(match[2].replace(/\\(.)/g, '$1'));
|
||||
}
|
||||
}
|
||||
return { names, dynamic };
|
||||
}
|
||||
|
||||
/**
|
||||
* Registry-time check of one guard: every name-filtered `node --test` must
|
||||
* match at least one static test name in its files. `read(file)` returns the
|
||||
* source or `null` for a missing file.
|
||||
*/
|
||||
export function staticTestSelectionProblems(guard, read) {
|
||||
const problems = [];
|
||||
for (const command of splitAndChain(guard)) {
|
||||
const selection = nodeTestSelection(command);
|
||||
if (!selection) continue;
|
||||
if (!selection.files.length || selection.files.some((file) => /[*?[]/.test(file))) continue;
|
||||
let regexps;
|
||||
try {
|
||||
regexps = selection.patterns.map(testNamePatternRegExp);
|
||||
} catch (error) {
|
||||
problems.push({ level: 'error', text: `--test-name-pattern не RegExp: ${error.message}` });
|
||||
continue;
|
||||
}
|
||||
const names = [];
|
||||
let dynamic = false;
|
||||
const missing = selection.files.filter((file) => read(file) == null);
|
||||
if (missing.length) {
|
||||
problems.push({ level: 'error', text: `нет файла ${missing.join(' ')}` });
|
||||
continue;
|
||||
}
|
||||
for (const file of selection.files) {
|
||||
const found = staticTestNames(read(file));
|
||||
names.push(...found.names);
|
||||
dynamic ||= found.dynamic;
|
||||
}
|
||||
if (names.some((name) => regexps.some((regexp) => regexp.test(name)))) continue;
|
||||
const text = `--test-name-pattern ${selection.patterns.map((p) => JSON.stringify(p)).join(', ')}`
|
||||
+ ` не совпадает ни с одним тестом в ${selection.files.join(' ')} (#650)`;
|
||||
problems.push(dynamic
|
||||
? { level: 'warn', text: `${text}; в файле есть имена с \${…} — проверить нельзя` }
|
||||
: { level: 'error', text });
|
||||
}
|
||||
return problems;
|
||||
}
|
||||
|
||||
export function classifyCommandResult(result, { phase = 'oracle', proof = MUTATION_PROOF.ASSERTION } = {}) {
|
||||
const stopped = interruption(result);
|
||||
if (stopped) return { ...stopped, command: result?.command || '' };
|
||||
if (Number(result.status) === 0) {
|
||||
// #650: a green oracle that executed no named test proves nothing either way.
|
||||
const empty = phase === 'oracle' ? emptyTestSelection(result?.command, outputOf(result)) : null;
|
||||
if (empty) return { kind: MUTATION_OUTCOME.SETUP, detail: empty, command: result?.command || '' };
|
||||
return { kind: MUTATION_OUTCOME.SURVIVED, detail: '', command: result?.command || '' };
|
||||
}
|
||||
const detail = outputOf(result).slice(-2000);
|
||||
|
||||
Reference in New Issue
Block a user