From c50458a0980aa974836abf5e17fe629634812aa7 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 21:56:40 +0300 Subject: [PATCH] ci: a stable release waits for a green E2E run on a real Home Assistant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stable gate proved Validate and Full Performance on the exact SHA but never ran the release in Home Assistant itself. houseplan-e2e installs the release's houseplan.zip — the bytes HACS ships — into HA in docker and walks the sidebar page, dashboards, roles, PDF, restart and the stable→tag upgrade. release.yml now dispatches e2e.yml on the tag for `!prerelease` releases and waits for it (scripts/e2e-gate.mjs, modelled on validate-gate.mjs): the gate recognises its own run by `HP ` in the job names, ignores foreign dispatches, and reports red / missing / cancelled / token error with the run link. Betas are untouched. Mutants: release-ships-on-red-e2e, release-trusts-foreign-e2e-run. Issue: #514 User-Visible: no --- .github/workflows/release.yml | 14 ++++ AGENTS.md | 5 +- PROCESS.md | 4 +- docs/DEVELOPMENT.md | 6 +- docs/TESTING.md | 9 +++ scripts/e2e-gate.mjs | 119 +++++++++++++++++++++++++++++++++ scripts/mutation-gate.mjs | 22 ++++++ test/e2e-gate.test.mjs | 108 ++++++++++++++++++++++++++++++ test/release-workflow.test.mjs | 25 +++++++ 9 files changed, 309 insertions(+), 3 deletions(-) create mode 100755 scripts/e2e-gate.mjs create mode 100755 test/e2e-gate.test.mjs create mode 100644 test/release-workflow.test.mjs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2af7e991..c31f6d2a 100755 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -50,6 +50,20 @@ jobs: set -euo pipefail SHA=$(git rev-parse HEAD) node scripts/release-gate.mjs "$SHA" --workflow=performance.yml --label="Полные бенчмарки производительности" + # #514: the only check on a real Home Assistant. houseplan-e2e installs + # the release's houseplan.zip — the bytes HACS ships — into HA in docker + # and walks the sidebar page, dashboards, roles, PDF, restart and the + # stable→tag upgrade. A red, missing or cancelled run withholds the + # assets exactly like Full Performance. Cross-repository dispatch needs a + # token with Actions: write on houseplan-e2e; HP_PROCESS_TOKEN (classic, + # repo scope) has it, E2E_DISPATCH_TOKEN is the fallback for a + # fine-grained token. + - name: Require green E2E on a real Home Assistant for a stable release + if: ${{ !github.event.release.prerelease }} + env: + GH_TOKEN: ${{ secrets.E2E_DISPATCH_TOKEN || secrets.HP_PROCESS_TOKEN }} + TAG: ${{ github.event.release.tag_name }} + run: node scripts/e2e-gate.mjs --tag="$TAG" build: name: Сборка бандла и загрузка ассетов needs: gate diff --git a/AGENTS.md b/AGENTS.md index 0566d2ac..6434a3cb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -464,7 +464,10 @@ an unfinished Validate for the same branch. Gate jobs, matching the actual `frontend`, `smoke`, `golden`, `performance_smoke`, `backend`. The `changes` job is a service path-filter, not a gate. `docs` is a real blocker: it checks the screenshots `sourceFingerprint` against current `src/**`, which is exactly what -went red after the #113 merge. +went red after the #113 merge. A stable release additionally waits for Full +Performance and for a green E2E run on a real Home Assistant (`houseplan-e2e`, +dispatched on the tag by `release.yml`, #514); betas and the development cycle +never run E2E. **"Verified" without a named command and its result is not evidence.** diff --git a/PROCESS.md b/PROCESS.md index b9eaa23c..2b3d1531 100644 --- a/PROCESS.md +++ b/PROCESS.md @@ -701,7 +701,9 @@ demo/docs/capture.mjs`, коммит вместе с задачей. достаточен для продолжения релиза, повторное код-ревью не требуется — §11.4. **Гейт стабильного релиза:** полный локальный прогон плюс Validate и Full -Performance зелёные на точном SHA; статусов issue не касается. +Performance зелёные на точном SHA, плюс зелёный E2E на реальном Home Assistant +на теге (`houseplan-e2e`, запускает и ждёт `release.yml`, #514); статусов +issue не касается. --- diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 9827bb85..fa4404cb 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -370,7 +370,11 @@ SHA to complete successfully (#511: a cancelled run is not a verdict, a later re-run or another-baseline comparison refreshes an older result), then builds and attaches `houseplan-card.js`. A missing, failed or one-hour-timed-out latest Validate withholds the asset; stable releases additionally need the same for -Full Performance. Bump the version +Full Performance and a green E2E run on a real Home Assistant: `release.yml` +dispatches `e2e.yml` in `Matysh/houseplan-e2e` with the tag (the suite installs +the release's `houseplan.zip` into HA in docker) and waits for it (#514). A red +E2E withholds the assets — open the linked run, the Playwright traces and +screenshots are in its artifacts; fix, then cut a new tag. Bump the version everywhere in sync: `src/houseplan-card.ts` (CARD_VERSION), `package.json`, `custom_components/houseplan/manifest.json`, `custom_components/houseplan/const.py`. diff --git a/docs/TESTING.md b/docs/TESTING.md index a97bed3f..f8f09924 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -65,6 +65,15 @@ dispatch-прогон на точном SHA; зелёный push-прогон и что отменённый пуш или таймаут не пропадают даром. Полный прогон и `--id` журнал не читают; `--ledger` без `--changed` — ошибка. +## E2E на реальном Home Assistant (#514) + +Репозиторий `Matysh/houseplan-e2e`: настоящий HA в docker, House Plan из +`houseplan.zip` релиза, 13 сценариев Playwright (боковая панель, дашборды, +роли, телефон, PDF, рестарт HA, первый запуск, обновление). Ночью — по +расписанию на последней бете; для **стабильного** релиза `release.yml` +запускает его на теге и ждёт зелёного (`scripts/e2e-gate.mjs`): красный — +ассеты не публикуются. В цикле разработки и на бетах не участвует. + ## Версия в кадрах и попиксельная приёмка (#512) Golden-кадры и скриншоты документации не должны меняться от bump версии. Для diff --git a/scripts/e2e-gate.mjs b/scripts/e2e-gate.mjs new file mode 100755 index 00000000..684c55c3 --- /dev/null +++ b/scripts/e2e-gate.mjs @@ -0,0 +1,119 @@ +#!/usr/bin/env node +/** + * E2E на реальном Home Assistant как гейт стабильного релиза (#514). + * + * Стабильный релиз проходил Validate и Full Performance на точном SHA, но ни + * разу не запускался в настоящем HA. Репозиторий houseplan-e2e ставит House + * Plan из `houseplan.zip` релиза — те же байты, что скачивает HACS, — и гоняет + * 13 сценариев Playwright. Этот скрипт запускает его workflow на теге и ждёт + * зелёного; `release.yml` вызывает его для `!prerelease` после Full Performance. + * + * node scripts/e2e-gate.mjs --tag= [--repo=Matysh/houseplan-e2e] [--workflow=e2e.yml] + * + * Печатает `result=green|red|missing|error`, `url=…`, `note=…` (и в + * $GITHUB_OUTPUT), код выхода 0 только при green. Логика — чистая функция + * `e2eGate` поверх инъектируемых `ops` (образец — validate-gate.mjs, #510). + */ +import { spawnSync } from 'node:child_process'; +import { appendFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { resolve } from 'node:path'; +import { VALIDATE_APPEAR_MS, VALIDATE_TOTAL_MS } from './merge-candidate.mjs'; + +export const POLL_MS = 20_000; +export const E2E_REPO = 'Matysh/houseplan-e2e'; +export const E2E_WORKFLOW = 'e2e.yml'; +/** Допуск на расхождение часов раннера и GitHub при отборе «свежих» прогонов. */ +export const CLOCK_SKEW_MS = 60_000; +export const TOKEN_HINT = 'нужен секрет E2E_DISPATCH_TOKEN с правом Actions: write на houseplan-e2e'; + +/** + * Прогон — наш, если хотя бы одна job названа по нашему тегу: имя job в + * e2e.yml — `"${suite} · HP ${ref} · HA ${ha}"`. Сьют `upgrade` носит + * `upgrade_from` (stable), но `journeys`/`first-run` несут тег — этого + * достаточно, чтобы не принять чужой dispatch (владелец запустил другой тег). + */ +export function isOurRun(jobs, tag) { + const needle = ` · HP ${tag} · `; + return (Array.isArray(jobs) ? jobs : []).some((job) => String(job?.name || '').includes(needle)); +} + +/** + * @param {object} p + * @param {string} p.tag тег релиза (houseplan_ref для e2e.yml) + * @param {object} p.ops { dispatch(tag), listRuns() → [{databaseId,status,conclusion,url,createdAt}], jobs(runId) → [{name,conclusion}], sleep(ms), now() } + * @returns {Promise<{result:'green'|'red'|'missing'|'error', url:string|null, note:string}>} + */ +export async function e2eGate({ tag, ops, appearMs = VALIDATE_APPEAR_MS, totalMs = VALIDATE_TOTAL_MS, pollMs = POLL_MS }) { + const started = ops.now(); + try { + await ops.dispatch(tag); + } catch (error) { + const message = String(error?.message || error); + const forbidden = /403|Resource not accessible|not accessible by/i.test(message); + return { result: 'error', url: null, note: `запуск e2e.yml не удался: ${message}${forbidden ? ` — ${TOKEN_HINT}` : ''}` }; + } + const foreign = new Set(); // dispatch-прогоны без нашего тега в именах job + let tracked = null; + while (ops.now() - started < totalMs) { + const runs = (await ops.listRuns()).filter((run) => !foreign.has(run.databaseId)); + let run = tracked ? runs.find((x) => x.databaseId === tracked) : null; + if (!run) { + // Опознание: свежий dispatch, чьи job носят наш тег. + for (const candidate of runs) { + const createdAt = Date.parse(candidate.createdAt || '') || 0; + if (createdAt < started - CLOCK_SKEW_MS) continue; + if (isOurRun(await ops.jobs(candidate.databaseId), tag)) { run = candidate; break; } + foreign.add(candidate.databaseId); + } + } + if (run) { + tracked = run.databaseId; + if (run.status === 'completed') { + if (run.conclusion === 'success') return { result: 'green', url: run.url, note: `E2E на ${tag} зелёный` }; + if (run.conclusion === 'cancelled') return { result: 'red', url: run.url, note: `E2E на ${tag} отменён вручную — перезапустите гейт` }; + return { result: 'red', url: run.url, note: `E2E на ${tag} завершился: ${run.conclusion}` }; + } + } else if (ops.now() - started > appearMs) { + return { result: 'missing', url: null, note: `dispatch e2e.yml на ${tag} не появился за ${Math.round(appearMs / 60000)} мин` }; + } + await ops.sleep(pollMs); + } + return { result: 'red', url: tracked ? `run ${tracked}` : null, note: `E2E на ${tag} не завершился за ${Math.round(totalMs / 60000)} мин` }; +} + +const sh = (cmd, args) => spawnSync(cmd, args, { encoding: 'utf8' }); + +export function realOps({ repo = E2E_REPO, workflow = E2E_WORKFLOW, exec = sh } = {}) { + const fields = 'databaseId,status,conclusion,url,createdAt'; + const parse = (r) => (r.status === 0 && r.stdout ? JSON.parse(r.stdout) : []); + return { + dispatch: async (tag) => { + const r = exec('gh', ['workflow', 'run', workflow, '--repo', repo, '--ref', 'main', + '-f', `houseplan_ref=${tag}`, '-f', 'upgrade_from=stable', '-f', 'ha_version=stable']); + if (r.status !== 0) throw new Error(`gh workflow run: ${(r.stderr || r.stdout || '').trim()}`); + }, + listRuns: async () => parse(exec('gh', ['run', 'list', '--repo', repo, '--workflow', workflow, '--event', 'workflow_dispatch', '--json', fields, '--limit', '10'])), + jobs: async (runId) => { + const r = exec('gh', ['run', 'view', String(runId), '--repo', repo, '--json', 'jobs']); + return r.status === 0 && r.stdout ? (JSON.parse(r.stdout).jobs || []).map((job) => ({ name: job.name, conclusion: job.conclusion })) : []; + }, + sleep: (ms) => new Promise((done) => setTimeout(done, ms)), + now: () => Date.now(), + }; +} + +const invokedDirectly = process.argv[1] && resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url)); +if (invokedDirectly) { + const arg = (name) => process.argv.find((a) => a.startsWith(`--${name}=`))?.slice(name.length + 3); + const tag = arg('tag'); + if (!tag) { + console.error('usage: e2e-gate.mjs --tag= [--repo=Matysh/houseplan-e2e] [--workflow=e2e.yml]'); + process.exit(2); + } + const outcome = await e2eGate({ tag, ops: realOps({ repo: arg('repo') || E2E_REPO, workflow: arg('workflow') || E2E_WORKFLOW }) }); + const lines = [`result=${outcome.result}`, `url=${outcome.url || ''}`, `note=${outcome.note}`]; + for (const line of lines) console.log(line); + if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `${lines.join('\n')}\n`); + process.exit(outcome.result === 'green' ? 0 : 1); +} diff --git a/scripts/mutation-gate.mjs b/scripts/mutation-gate.mjs index 50ef66a6..8a496e53 100644 --- a/scripts/mutation-gate.mjs +++ b/scripts/mutation-gate.mjs @@ -8214,6 +8214,28 @@ const MUTANT_DEFINITIONS = [ replace: " const runs = all.filter((x) => (!event || x.event === event)); // mutant: cancelled is red", }], }, + { + id: 'release-ships-on-red-e2e', + guard: 'node --test test/e2e-gate.test.mjs', + because: 'a stable release must wait for a green E2E on a real Home Assistant; a gate that reads ' + + 'a failed run as green ships the assets the run just rejected (#514 AC1)', + patches: [{ + file: 'scripts/e2e-gate.mjs', + find: " if (run.conclusion === 'success') return { result: 'green', url: run.url, note: `E2E на ${tag} зелёный` };", + replace: " return { result: 'green', url: run.url, note: `E2E на ${tag} зелёный` }; // mutant: completed means green", + }], + }, + { + id: 'release-trusts-foreign-e2e-run', + guard: 'node --test test/e2e-gate.test.mjs', + because: 'the gate must follow the dispatch it made for this tag; accepting any dispatch run lets ' + + 'a green run on another tag vouch for this release (#514 AC2)', + patches: [{ + file: 'scripts/e2e-gate.mjs', + find: " return (Array.isArray(jobs) ? jobs : []).some((job) => String(job?.name || '').includes(needle));", + replace: " return true; // mutant: every dispatch is ours", + }], + }, { id: 'review-trusts-push-run-without-mutants', guard: 'node --test test/validate-gate.test.mjs', diff --git a/test/e2e-gate.test.mjs b/test/e2e-gate.test.mjs new file mode 100755 index 00000000..d790ac07 --- /dev/null +++ b/test/e2e-gate.test.mjs @@ -0,0 +1,108 @@ +// #514: a stable release waits for a green E2E run on a real Home Assistant. +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { e2eGate, isOurRun, realOps, TOKEN_HINT } from '../scripts/e2e-gate.mjs'; + +const TAG = 'v1.74.0'; +const ours = (suffix = '') => [{ name: `journeys · HP ${TAG} · HA stable${suffix}`, conclusion: 'success' }, { name: 'upgrade · HP stable · HA stable', conclusion: 'success' }]; +const theirs = () => [{ name: 'journeys · HP v1.73.0 · HA stable', conclusion: 'success' }]; + +/** Fake gh: scripted run-list snapshots per poll, jobs per run id, a virtual clock. */ +function fakeOps({ snapshots, jobsById = {}, dispatchError = null, startedAt = 100_000 }) { + let clock = startedAt; + let calls = 0; + const dispatched = []; + return { + ops: { + dispatch: async (tag) => { if (dispatchError) throw new Error(dispatchError); dispatched.push(tag); }, + listRuns: async () => { const s = snapshots[Math.min(calls, snapshots.length - 1)]; calls += 1; return s; }, + jobs: async (id) => jobsById[id] ?? [], + sleep: async (ms) => { clock += ms; }, + now: () => clock, + }, + dispatched, + calls: () => calls, + }; +} + +const run = (over) => ({ databaseId: 1, status: 'completed', conclusion: 'success', url: 'https://e2e/run/1', createdAt: new Date(100_500).toISOString(), ...over }); + +test('#514 AC2: a run is ours only when a job carries our tag', () => { + assert.equal(isOurRun(ours(), TAG), true); + assert.equal(isOurRun(theirs(), TAG), false); + assert.equal(isOurRun([{ name: `journeys · HP ${TAG}-beta.1 · HA stable` }], TAG), false, 'a prerelease of the same version is not the tag'); + assert.equal(isOurRun([], TAG), false); +}); + +test('#514 AC1: dispatch, then the green run on the tag is accepted', async () => { + const fake = fakeOps({ snapshots: [[run({ status: 'in_progress', conclusion: null })], [run()]], jobsById: { 1: ours() } }); + const outcome = await e2eGate({ tag: TAG, ops: fake.ops, pollMs: 1000 }); + assert.equal(outcome.result, 'green'); + assert.equal(outcome.url, 'https://e2e/run/1'); + assert.deepEqual(fake.dispatched, [TAG], 'exactly one dispatch with the release tag'); +}); + +test('#514 AC1: a red run withholds the assets and names the run', async () => { + const fake = fakeOps({ snapshots: [[run({ conclusion: 'failure', url: 'https://e2e/run/red' })]], jobsById: { 1: ours() } }); + const outcome = await e2eGate({ tag: TAG, ops: fake.ops, pollMs: 1000 }); + assert.equal(outcome.result, 'red'); + assert.equal(outcome.url, 'https://e2e/run/red'); + assert.match(outcome.note, /failure/); +}); + +test('#514 AC2: a foreign dispatch on another tag is ignored, ours is tracked even when it appears later', async () => { + const foreign = run({ databaseId: 7, url: 'https://e2e/run/7' }); + const mine = run({ databaseId: 9, url: 'https://e2e/run/9' }); + const fake = fakeOps({ snapshots: [[foreign], [foreign], [mine, foreign]], jobsById: { 7: theirs(), 9: ours() } }); + const outcome = await e2eGate({ tag: TAG, ops: fake.ops, pollMs: 1000 }); + assert.equal(outcome.result, 'green'); + assert.equal(outcome.url, 'https://e2e/run/9'); +}); + +test('#514 AC2: a stale run from before the dispatch is not ours even with the same tag', async () => { + const stale = run({ databaseId: 3, url: 'https://e2e/run/stale', createdAt: new Date(100_000 - 120_000).toISOString() }); + const fake = fakeOps({ snapshots: [[stale]], jobsById: { 3: ours() } }); + const outcome = await e2eGate({ tag: TAG, ops: fake.ops, appearMs: 5000, pollMs: 1000 }); + assert.equal(outcome.result, 'missing'); +}); + +test('#514 AC1: no run appears within the appear window → missing', async () => { + const fake = fakeOps({ snapshots: [[]] }); + const outcome = await e2eGate({ tag: TAG, ops: fake.ops, appearMs: 5000, pollMs: 1000 }); + assert.equal(outcome.result, 'missing'); + assert.match(outcome.note, /не появился/); +}); + +test('#514 AC1: a run that never finishes is red after the total window', async () => { + const fake = fakeOps({ snapshots: [[run({ status: 'in_progress', conclusion: null })]], jobsById: { 1: ours() } }); + const outcome = await e2eGate({ tag: TAG, ops: fake.ops, totalMs: 10_000, pollMs: 4000 }); + assert.equal(outcome.result, 'red'); + assert.match(outcome.note, /не завершился/); +}); + +test('#514 AC1: a cancelled run is red with an explicit note — nobody cancels e2e.yml but a hand', async () => { + const fake = fakeOps({ snapshots: [[run({ conclusion: 'cancelled' })]], jobsById: { 1: ours() } }); + const outcome = await e2eGate({ tag: TAG, ops: fake.ops, pollMs: 1000 }); + assert.equal(outcome.result, 'red'); + assert.match(outcome.note, /отменён вручную/); +}); + +test('#514 AC1: a dispatch refused by the token is an error that names the missing secret', async () => { + const fake = fakeOps({ snapshots: [[]], dispatchError: 'gh workflow run: HTTP 403: Resource not accessible by personal access token' }); + const outcome = await e2eGate({ tag: TAG, ops: fake.ops, pollMs: 1000 }); + assert.equal(outcome.result, 'error'); + assert.ok(outcome.note.includes(TOKEN_HINT), outcome.note); + assert.equal(fake.calls(), 0, 'no polling after a failed dispatch'); +}); + +test('#514: realOps dispatches e2e.yml on main with the tag and the stable baseline', async () => { + const calls = []; + const exec = (cmd, args) => { calls.push([cmd, ...args]); return { status: 0, stdout: '[]', stderr: '' }; }; + const ops = realOps({ exec }); + await ops.dispatch(TAG); + assert.deepEqual(calls[0], ['gh', 'workflow', 'run', 'e2e.yml', '--repo', 'Matysh/houseplan-e2e', '--ref', 'main', + '-f', `houseplan_ref=${TAG}`, '-f', 'upgrade_from=stable', '-f', 'ha_version=stable']); + await ops.listRuns(); + assert.ok(calls[1].includes('--event') && calls[1].includes('workflow_dispatch')); +}); diff --git a/test/release-workflow.test.mjs b/test/release-workflow.test.mjs new file mode 100644 index 00000000..78de1b86 --- /dev/null +++ b/test/release-workflow.test.mjs @@ -0,0 +1,25 @@ +// #514: release.yml holds the assets of a stable release until E2E on a real HA is green. +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { readFileSync } from 'node:fs'; + +const workflow = readFileSync(new URL('../.github/workflows/release.yml', import.meta.url), 'utf8'); +const at = (marker) => { const i = workflow.indexOf(marker); assert.ok(i > 0, `нет «${marker}»`); return i; }; + +test('#514 AC1/AC3: the E2E gate step exists in job gate, after Full Performance, for stable releases only', () => { + const gate = at(' gate:\n'); + const build = at(' build:\n'); + const perf = at(' - name: Require full performance for a stable release\n'); + const e2e = at(' - name: Require green E2E on a real Home Assistant for a stable release\n'); + assert.ok(gate < perf && perf < e2e && e2e < build, 'E2E stands after Full Performance inside job gate'); + const step = workflow.slice(e2e, build); + assert.match(step, /if: \$\{\{ !github\.event\.release\.prerelease \}\}/, 'prereleases skip the step'); + assert.match(step, /node scripts\/e2e-gate\.mjs --tag="\$TAG"/); + assert.match(step, /TAG: \$\{\{ github\.event\.release\.tag_name \}\}/); +}); + +test('#514: the gate dispatches with a token that can reach houseplan-e2e, with the process token as fallback', () => { + const e2e = at(' - name: Require green E2E on a real Home Assistant for a stable release\n'); + const step = workflow.slice(e2e, at(' build:\n')); + assert.match(step, /GH_TOKEN: \$\{\{ secrets\.E2E_DISPATCH_TOKEN \|\| secrets\.HP_PROCESS_TOKEN \}\}/); +});