Files
houseplan-card/test/release-workflow.test.mjs
T
Claudeandclaude[bot] cb97274ea7 test(harness): run workflow steps the way the runner does (#766)
Thirteen test harnesses executed workflow `run:` bodies with their own bash
flags. Four of them used `bash -eo pipefail` "as in Actions", but the runner
executes a step without `shell:` as `bash -e {0}`: pipefail comes only from an
explicit `shell: bash` or from `set -o pipefail` in the body. The harness
supplied protection the step did not have, so a step that lost its pipefail
stayed green in tests (#729, #737, #751); the reverse also happened - the
#793 guard test was red only because of the harness flag.

test/helpers/workflow-step.mjs resolves the shell like the runner (step ->
jobs.<id>.defaults.run.shell -> workflow defaults.run.shell -> unset), maps it
to the runner's command lines (unset -> `bash -e {0}`, bash -> `bash
--noprofile --norc -e -o pipefail {0}`, sh, python, custom templates with
{0}), writes the body to a file and executes it by path. Unsupported YAML or
shells are refused loudly instead of guessed. All step-executing tests now go
through runStep(findStep(...)).

Witnesses: a step whose left pipeline side fails is green without a shell
(negative test) and red with `shell: bash`, job defaults or `set -o pipefail`;
the #751 executed test now also shows that the same real steps without their
pipefail line stay green, i.e. the test sees a removed pipefail; a guard fails
on any test that runs a step with its own bash flags. TESTING.md rule 7 names
the helper.

Issue: #766
User-Visible: no
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018qZfe7YS4rqEMKoVeS3GKd
2026-10-06 23:05:58 +00:00

257 lines
18 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// #514: release.yml holds the assets of a stable release until E2E on a real HA is green.
// #540: release.yml is the ONLY publisher of installable assets, and it publishes
// only after the gates saw the very same bytes.
import assert from 'node:assert/strict';
import test from 'node:test';
import { spawnSync } from 'node:child_process';
import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { findStep, runStep } from './helpers/workflow-step.mjs';
const WORKFLOWS = fileURLToPath(new URL('../.github/workflows/', import.meta.url));
const read = (name) => readFileSync(new URL(name, `file://${WORKFLOWS}`), 'utf8');
const workflow = read('release.yml');
const at = (marker, text = workflow) => { const i = text.indexOf(marker); assert.ok(i > 0, `нет «${marker}»`); return i; };
const job = (name) => {
const start = at(`\n ${name}:\n`);
const rest = workflow.slice(start + 1);
const next = rest.slice(1).search(/\n {2}[a-z-]+:\n/);
return next < 0 ? rest : rest.slice(0, next + 1);
};
const jobNeeds = (name) => {
const m = /^ {4}needs: (.+)$/m.exec(job(name));
if (!m) return [];
return m[1].replace(/[[\]\s]/g, '').split(',').filter(Boolean);
};
test('#540 AC1: exactly one workflow reacts to the release event, and none of them publishes on it', () => {
const listeners = readdirSync(WORKFLOWS).filter((name) => name.endsWith('.yml'))
.filter((name) => /^\s*release:\s*\n\s+types:/m.test(read(name).slice(0, read(name).indexOf('\njobs:'))));
assert.deepEqual(listeners, ['release.yml'], 'release-zip.yml (immediate ZIP upload) is gone and must not come back');
assert.ok(!readdirSync(WORKFLOWS).includes('release-zip.yml'));
// asset uploads live only in the job that needs the gate
const jobs = [...workflow.slice(at('\njobs:\n')).matchAll(/^ {2}([a-z-]+):\n/gm)].map((m) => m[1]);
assert.deepEqual(jobs, ['candidate', 'independent-review', 'gate', 'stage', 'publish', 'announce', 'hacs-discovery']);
const uploads = jobs.filter((name) => /gh release upload|softprops\/action-gh-release/.test(job(name)));
assert.deepEqual(uploads, ['stage'], 'the one uploading job');
assert.deepEqual(jobNeeds('stage'), ['candidate', 'gate']);
assert.deepEqual(jobNeeds('publish'), ['candidate', 'gate', 'stage']);
});
test('#540 AC2: a release published by hand is taken back to draft before any gate runs', () => {
const candidate = job('candidate');
assert.match(candidate, /gh release edit "\$TAG" --repo "\$GITHUB_REPOSITORY" --draft\n/, 'fail-closed re-draft');
assert.ok(at('--draft\n', candidate) < at('\n gate:\n'), 're-draft is in the candidate job, ahead of the gate');
assert.match(candidate, /if \[ "\$EVENT" = "release" \]; then/, 'only a hand-made publication is re-drafted');
assert.match(candidate, /echo "mode=repair"/, 'a dispatch on a public release is a repair, not a re-publication');
assert.match(candidate, /if: \$\{\{ github\.event_name == 'workflow_dispatch' \|\| !github\.event\.release\.prerelease \}\}/,
'betas published by hand are skipped: they have their own staged path');
const triggers = workflow.slice(at('\non:\n'), at('\npermissions:'));
assert.match(triggers, /release:\n\s+types: \[published\]/, '`created` never fires for drafts — `published` catches both paths');
assert.match(triggers, /workflow_dispatch:\n\s+inputs:\n\s+tag:/);
});
test('#540 AC1/#514: the gate judges the exact SHA — trailer names the tag, contract, Validate, Full Performance, E2E on the SHA', () => {
const gate = job('gate');
const trailer = at('grep -Fxq "Release: $TAG"', gate);
const contract = at('node scripts/release-contract.mjs "$TAG" --repo="$GITHUB_REPOSITORY" --stable', gate);
const validate = at('node scripts/release-gate.mjs "$SHA"\n', gate);
const perf = at(' - name: Require full performance for a stable release\n', gate);
const e2e = at(' - name: Require green E2E on a real Home Assistant for a stable release\n', gate);
assert.ok(trailer < contract && contract < validate && validate < perf && perf < e2e, 'order: trailer, contract, Validate, Full Performance, E2E');
const e2eStep = gate.slice(e2e);
assert.match(e2eStep, /if: \$\{\{ needs\.candidate\.outputs\.prerelease != 'true' \}\}/, 'prereleases skip the step');
assert.match(e2eStep, /node scripts\/e2e-gate\.mjs --ref="\$SHA" --tag="\$TAG"/, 'E2E installs the candidate tree, not a public ZIP');
assert.match(e2eStep, /GH_TOKEN: \$\{\{ secrets\.E2E_DISPATCH_TOKEN \|\| secrets\.HP_PROCESS_TOKEN \}\}/);
assert.match(gate, /--workflow=performance\.yml --label="Полные бенчмарки производительности"/);
assert.ok(!/github\.event\.release\.tag_name/.test(gate + job('stage') + job('publish')), 'every job works from the resolved candidate, not the event payload');
});
test('#540 AC3: one build, deterministic ZIP from the tree E2E installed, passport, verified public bytes', () => {
const stage = job('stage');
assert.match(stage, /git -c core\.autocrlf=false archive --format=zip --output=houseplan\.zip \\\n\s+"\$SHA:custom_components\/houseplan"/);
assert.match(stage, /node scripts\/verify-houseplan-zip\.mjs houseplan\.zip/);
assert.match(stage, /git rev-parse "\$SHA:custom_components\/houseplan"/, 'tree hash printed: identity with the E2E tarball');
assert.match(stage, /node scripts\/release-assets\.mjs sums release-assets/);
assert.match(stage, /test -s dist\/houseplan-panel\.js/);
assert.ok(at('node scripts/release-assets.mjs sums', stage) < at('gh release upload', stage), 'passport before upload');
// repair: only missing assets, a differing hash is a failure
assert.match(stage, /if \[ "\$MODE" = "repair" \]; then/);
assert.match(stage, /node scripts\/release-assets\.mjs check public release-assets\/SHA256SUMS --allow-missing/);
const repair = stage.slice(at('if [ "$MODE" = "repair" ]', stage), at(' else\n # Draft', stage));
const repairCommands = repair.split('\n').filter((line) => !/^\s*#/.test(line) && !/gh release download/.test(line)).join('\n');
assert.ok(!/--clobber/.test(repairCommands), 'repair never clobbers a public asset');
assert.match(repair, /gh release upload "\$TAG" \$missing --repo "\$GITHUB_REPOSITORY"\n/);
const publish = job('publish');
assert.ok(at('--draft=false', publish) < at('gh release download', publish), 'publish, then read back what the public sees');
assert.match(publish, /diff -u passport\/SHA256SUMS public\/SHA256SUMS/);
assert.match(publish, /node scripts\/release-assets\.mjs check public passport\/SHA256SUMS\n/);
assert.match(publish, /test "\$\(git rev-list -n 1 "refs\/tags\/\$TAG"\)" = "\$SHA"/);
assert.match(publish, /download-artifact@/, 'the passport travels from stage as an artifact, not via the release');
});
// #538: анонс — последнее звено выпуска, а не параллельное ему. Пока он висел
// на самом событии `release: published`, гонку он выигрывал всегда: проверять
// ему нечего. 12.09 v1.75.0 объявили в канале в ту же минуту, когда гейт
// отказал выкладывать ассеты, и снаружи это выглядело обычным релизом.
const announce = read('announce.yml');
test('#538 AC1: событие релиза не может запустить анонс', () => {
const triggers = announce.slice(announce.indexOf('\non:'), announce.indexOf('\npermissions:'));
assert.ok(!/^\s*release:/m.test(triggers), 'в триггерах анонса нет `release:`');
assert.match(triggers, /^\s*workflow_dispatch:/m, 'кнопка проверки связи остаётся');
assert.match(triggers, /^\s*workflow_call:/m, 'вызов из воркфлоу остаётся');
assert.ok(!/github\.event\.release\./.test(announce),
'мёртвая ветка события не оставлена в шагах');
});
test('#538 AC2 / #540: release.yml зовёт анонс только после публикации проверенных ассетов', () => {
const block = job('announce');
assert.ok(at('\n publish:\n') < at('\n announce:\n'), 'анонс описан после публикации, а не до неё');
// Не `/needs: publish/`: в том же блоке лежит комментарий, где эта строка
// процитирована, и проверка зеленела бы на нём. Требуется сама директива.
assert.match(block, /^ {4}needs: \[candidate, publish\]$/m, 'анонс зависит от публикации');
assert.match(block, /if: \$\{\{ needs\.publish\.outputs\.newly_published == 'true' \}\}/, 'ремонт не анонсируется');
assert.match(block, /uses: \.\/\.github\/workflows\/announce\.yml/);
assert.match(block, /prerelease: \$\{\{ needs\.candidate\.outputs\.prerelease == 'true' \}\}/,
'беты остаются тихими по признаку тега');
assert.match(block, /secrets: inherit/);
});
// #638, PROCESS.md §11.5: независимое ревью линии запускается параллельно и
// выпуск не блокирует (решение владельца 2026-09-25). Ни один job выпуска не
// может зависеть от него: иначе «рекомендация» молча превращается в гейт.
test('#638 AC2: ревью линии ставится в очередь параллельно гейтам и ни один job выпуска его не ждёт', () => {
const block = job('independent-review');
assert.deepEqual(jobNeeds('independent-review'), ['candidate'], 'стартует сразу после закрепления SHA');
assert.match(block, /^ {4}continue-on-error: true$/m, 'отказ запуска — не красный релиз');
assert.match(block, /if: \$\{\{ needs\.candidate\.outputs\.prerelease != 'true' \}\}/, 'только стабильные');
assert.match(block, /gh workflow run release-review\.yml --repo "\$\{\{ github\.repository \}\}" --ref dev/);
assert.match(block, /-f tag="\$TAG" -f candidate="\$SHA"/, 'ревью судит тот же SHA, что гейты');
assert.match(block, /^ {4}permissions:\n {6}actions: write\n {4}steps:/m, 'единственное право — поставить workflow в очередь');
const jobs = [...workflow.slice(at('\njobs:\n')).matchAll(/^ {2}([a-z-]+):\n/gm)].map((m) => m[1]);
for (const name of jobs) {
assert.ok(!jobNeeds(name).includes('independent-review'), `${name} не зависит от ревью линии`);
assert.ok(!/needs\.independent-review/.test(job(name)), `${name} не читает результат ревью линии`);
}
});
// #704: dispatch не возвращает прогона — v1.78.0 выпустился, а ревью линии
// упало в прогоне, о котором выпуск не знал (release run 36468444979, ревью
// 36468505112). Job находит поставленный прогон и пишет ссылку и статус в
// сводку; не нашёл за несколько минут — предупреждение, выпуск не блокируется.
// Шаг исполняется настоящим bash по тексту из release.yml; gh и sleep подменены.
const reviewStepScript = () => {
const block = job('independent-review');
const lines = block.split('\n');
const runAt = lines.findIndex((line) => /^ {8}run: \|\s*$/.test(line));
assert.ok(runAt > 0, 'у шага есть run: |');
const body = [];
for (const line of lines.slice(runAt + 1)) {
if (line.trim() && !line.startsWith(' ')) break;
body.push(line.slice(10));
}
return body.join('\n').replace(/\$\{\{ github\.repository \}\}/g, 'o/r');
};
const hasTools = () => process.platform !== 'win32'
&& ['bash', 'jq'].every((tool) => spawnSync(tool, ['--version']).status === 0);
/**
* `snapshots` — ответы `gh run list` по порядку опросов (последний повторяется),
* `dispatch` — код `gh workflow run`.
*/
function runReviewStep({ snapshots = [[]], dispatch = 0, appear = 45, poll = 15 } = {}) {
const dir = mkdtempSync(join(tmpdir(), 'hp-704-'));
try {
const bin = join(dir, 'bin');
mkdirSync(bin);
snapshots.forEach((rows, i) => writeFileSync(join(dir, `runs-${i + 1}.json`), JSON.stringify(rows)));
writeFileSync(join(bin, 'gh'), [
'#!/bin/bash',
`dir=${JSON.stringify(dir)}`,
'echo "$*" >> "$dir/gh.log"',
`if [ "$1 $2" = "workflow run" ]; then exit ${dispatch}; fi`,
'if [ "$1 $2" = "run list" ]; then',
' n=$(( $(cat "$dir/n" 2>/dev/null || echo 0) + 1 )); echo "$n" > "$dir/n"',
` f="$dir/runs-$n.json"; [ -f "$f" ] || f="$dir/runs-${snapshots.length}.json"`,
' cat "$f"; exit 0',
'fi',
'echo "unexpected gh $*" >&2; exit 97',
'',
].join('\n'), { mode: 0o755 });
writeFileSync(join(bin, 'sleep'), '#!/bin/sh\nexit 0\n', { mode: 0o755 });
const summary = join(dir, 'summary.md');
writeFileSync(summary, '');
// #766: shell шага — по правилам раннера (без `shell:` — `bash -e {0}`).
const step = findStep(workflow, { name: 'Поставить в очередь ревью линии' }, 'release.yml');
assert.equal(step.job, 'independent-review');
const r = runStep(step, reviewStepScript(), {
encoding: 'utf8',
env: {
...process.env, PATH: `${bin}:${process.env.PATH}`, GH_TOKEN: 'x', TAG: 'v1.79.0', SHA: 'c'.repeat(40),
APPEAR_SECONDS: String(appear), POLL_SECONDS: String(poll), GITHUB_STEP_SUMMARY: summary,
},
});
const log = (() => { try { return readFileSync(join(dir, 'gh.log'), 'utf8'); } catch { return ''; } })();
return { status: r.status, stdout: r.stdout, stderr: r.stderr, summary: readFileSync(summary, 'utf8'), gh: log.split('\n').filter(Boolean) };
} finally {
rmSync(dir, { recursive: true, force: true });
}
}
const iso = (offsetSeconds) => new Date(Date.now() + offsetSeconds * 1000).toISOString().replace(/\.\d+Z$/, 'Z');
const reviewRun = (id, status, extra = {}) => ({
databaseId: id, url: `https://github.com/o/r/actions/runs/${id}`, status, conclusion: '',
displayTitle: 'Release review v1.79.0', createdAt: iso(0), ...extra,
});
test('#704 AC2: прогон ревью найден и стартовал — ссылка и статус в сводке, без предупреждения', { skip: !hasTools() && 'нужны bash и jq' }, () => {
const older = reviewRun(1, 'completed', { conclusion: 'failure', createdAt: iso(-3600) });
const otherTag = reviewRun(2, 'in_progress', { displayTitle: 'Release review v1.78.0' });
const r = runReviewStep({ snapshots: [[older, otherTag], [older, otherTag, reviewRun(3, 'queued')], [older, otherTag, reviewRun(3, 'in_progress')]] });
assert.equal(r.status, 0, r.stderr);
assert.doesNotMatch(r.stdout, /::warning::/);
assert.match(r.summary, /Независимое ревью v1\.79\.0 \*\*запущено\*\*: \[прогон\]\(https:\/\/github\.com\/o\/r\/actions\/runs\/3\), статус in_progress\. Выпуск его не ждёт\./);
const lists = r.gh.filter((line) => line.startsWith('run list'));
assert.equal(lists.length, 3, 'опрос остановился, как только прогон стартовал');
assert.match(lists[0], /--workflow release-review\.yml --branch dev --event workflow_dispatch/);
assert.match(r.gh[0], /^workflow run release-review\.yml --repo o\/r --ref dev -f tag=v1\.79\.0 -f candidate=c{40}$/, 'dispatch — до поиска');
});
test('#704 AC2: прогон не появился — предупреждение «не стартовало за N мин», шаг не красный', { skip: !hasTools() && 'нужны bash и jq' }, () => {
const stale = reviewRun(1, 'completed', { conclusion: 'failure', createdAt: iso(-3600) });
const r = runReviewStep({ snapshots: [[stale]], appear: 180, poll: 15 });
assert.equal(r.status, 0, 'выпуск не блокируется');
assert.match(r.stdout, /^::warning::прогон ревью линии v1\.79\.0 не появился за 3 мин/m);
assert.match(r.summary, /Независимое ревью v1\.79\.0: \*\*не стартовало за 3 мин\*\*/);
assert.equal(r.gh.filter((line) => line.startsWith('run list')).length, 12, 'опрос ограничен окном: 180 с / 15 с');
assert.doesNotMatch(r.summary, /runs\/1/, 'прогон прошлого запуска — не этот');
});
test('#704 AC2: прогон в очереди всё окно — предупреждение со ссылкой; отказ dispatch — прежний', { skip: !hasTools() && 'нужны bash и jq' }, () => {
const queued = runReviewStep({ snapshots: [[reviewRun(5, 'queued')]], appear: 30, poll: 15 });
assert.equal(queued.status, 0);
assert.match(queued.stdout, /^::warning::ревью линии v1\.79\.0 в очереди и не стартовало за 1 мин: https:\/\/github\.com\/o\/r\/actions\/runs\/5$/m);
assert.match(queued.summary, /\*\*не стартовало за 1 мин\*\* \(в очереди\) — \[прогон\]\(https:\/\/github\.com\/o\/r\/actions\/runs\/5\), статус queued\./);
const refused = runReviewStep({ dispatch: 1 });
assert.equal(refused.status, 1, 'job с continue-on-error: отказ dispatch виден, выпуск идёт');
assert.match(refused.stdout, /^::warning::ревью линии v1\.79\.0 не запущено — выпуск продолжается/m);
assert.match(refused.summary, /\*\*не запущено\*\*/);
assert.ok(!refused.gh.some((line) => line.startsWith('run list')), 'без dispatch искать нечего');
});
test('#704 AC2: ожидание прогона укладывается в бюджет job', () => {
const block = job('independent-review');
const appear = Number(/^ {10}APPEAR_SECONDS: (\d+)$/m.exec(block)?.[1]);
const poll = Number(/^ {10}POLL_SECONDS: (\d+)$/m.exec(block)?.[1]);
const timeout = Number(/^ {4}timeout-minutes: (\d+)$/m.exec(block)?.[1]);
assert.ok(appear > 0 && poll > 0 && timeout > 0, JSON.stringify({ appear, poll, timeout }));
assert.ok(appear <= 5 * 60, 'ждать не дольше нескольких минут');
assert.ok(appear + 60 < timeout * 60, `окно ${appear} с + запас на dispatch и опросы < timeout ${timeout} мин`);
});