From 9d8f89d2600d573a7d57834659c1de0f0445f2dd Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 17:38:48 +0300 Subject: [PATCH] ci: release gate judges the latest non-cancelled Validate run of the SHA MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gate used to require every Validate run on the tag SHA to be green: a cancelled duplicate or a red flake that a later re-run had fixed kept the stable release blocked (v1.73.0, 09.09 — released by hand). Now the verdict comes from the newest run that was not cancelled: not completed → wait, success → pass, anything else → fail, no run → wait. The same rule is documented for the perf workflow and the release runbook. Mutants: release-gate-counts-cancelled-runs, release-gate-oldest-run-wins. Issue: #511 User-Visible: no --- demo/performance/README.md | 4 +++- docs/DEVELOPMENT.md | 9 ++++++--- scripts/mutation-gate.mjs | 22 +++++++++++++++++++++ scripts/release-gate.mjs | 33 ++++++++++++++++++++++--------- test/release-gate.test.mjs | 40 ++++++++++++++++++++++++++++---------- 5 files changed, 85 insertions(+), 23 deletions(-) diff --git a/demo/performance/README.md b/demo/performance/README.md index 8d009ae0..44d58112 100644 --- a/demo/performance/README.md +++ b/demo/performance/README.md @@ -121,7 +121,9 @@ one runner. Its raw reports and comparison are uploaded as `full-performance-`, and the table is written to that GitHub job summary. Stable release assets require both exact-SHA `Validate` and exact-SHA `Full Performance`; prereleases require only -`Validate`. +`Validate`. The gate (`scripts/release-gate.mjs`, #511) judges by the latest +non-cancelled run on the SHA: a re-run or a manual comparison against another +baseline refreshes the verdict, and a cancelled twin is invisible. This base-vs-candidate design intentionally does not compare timings captured on different machines or different Chromium builds. A runtime/profile mismatch diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 9193d8f4..0266c180 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -355,9 +355,12 @@ fallback. They still gate assets on the exact tagged SHA, so adopting the new path does not weaken releases created through the old path. Tag `vX.Y.Z` + GitHub Release → `.github/workflows/release.yml` resolves that -tag to its exact commit, waits for every Validate run of the SHA to complete -successfully, then builds and attaches `houseplan-card.js`. A missing, failed, -cancelled or one-hour-timed-out Validate withholds the asset. Bump the version +tag to its exact commit, waits for the latest non-cancelled Validate run of the +SHA to complete successfully (#511: a cancelled run is not a verdict, a later +re-run or another-baseline comparison refreshes an older result), then builds +and attaches `houseplan-card.js`. A missing, failed or one-hour-timed-out latest +Validate withholds the asset; stable releases additionally need the same for +Full Performance. Bump the version everywhere in sync: `src/houseplan-card.ts` (CARD_VERSION), `package.json`, `custom_components/houseplan/manifest.json`, `custom_components/houseplan/const.py`. diff --git a/scripts/mutation-gate.mjs b/scripts/mutation-gate.mjs index bbcfd39e..229592a4 100644 --- a/scripts/mutation-gate.mjs +++ b/scripts/mutation-gate.mjs @@ -8087,6 +8087,28 @@ const MUTANT_DEFINITIONS = [ + ' # The store is the durable authority (#335): a drop that\n', }], }, + { + id: 'release-gate-counts-cancelled-runs', + guard: 'node --test test/release-gate.test.mjs', + because: 'a cancelled twin on the tag SHA proves nothing and must not block the assets; the ' + + 'verdict is the latest non-cancelled run (#511 AC1)', + patches: [{ + file: 'scripts/release-gate.mjs', + find: " const relevant = (Array.isArray(runs) ? runs : []).filter((run) => run && run.conclusion !== 'cancelled');", + replace: " const relevant = (Array.isArray(runs) ? runs : []).filter((run) => !!run); // mutant: cancelled counts", + }], + }, + { + id: 'release-gate-oldest-run-wins', + guard: 'node --test test/release-gate.test.mjs', + because: 'the latest run is the verdict: a re-run or another-baseline comparison must be able to ' + + 'refresh an older result on the same SHA (#511 AC1)', + patches: [{ + file: 'scripts/release-gate.mjs', + find: " return relevant.sort((a, b) => stamp(b) - stamp(a) || Number(b.id || 0) - Number(a.id || 0))[0] || null;", + replace: " return relevant.sort((a, b) => stamp(a) - stamp(b) || Number(a.id || 0) - Number(b.id || 0))[0] || null; // mutant: oldest", + }], + }, { id: 'summary-runtime-attaches-after-first-render', guard: 'node demo/smoke_summary_warm_attach.mjs', diff --git a/scripts/release-gate.mjs b/scripts/release-gate.mjs index a2d9cf1d..c86d3a64 100644 --- a/scripts/release-gate.mjs +++ b/scripts/release-gate.mjs @@ -4,11 +4,26 @@ import { resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; +/** + * The verdict is the LATEST run that was not cancelled (#511). A cancelled run + * proves nothing either way — concurrency or a hand superseded it — and the + * old "every run must be green" rule turned one red or cancelled duplicate on + * a SHA into a permanent block (v1.73.0: a cancelled dispatch twin plus a red + * comparison next to a green push run left the tag without assets). A later + * re-run or a dispatch with another baseline may therefore refresh the verdict + * on the same SHA; the owner accepted that trade-off deliberately. + */ +export function latestRelevantRun(runs) { + const relevant = (Array.isArray(runs) ? runs : []).filter((run) => run && run.conclusion !== 'cancelled'); + const stamp = (run) => Date.parse(run.run_started_at || run.created_at || 0) || 0; + return relevant.sort((a, b) => stamp(b) - stamp(a) || Number(b.id || 0) - Number(a.id || 0))[0] || null; +} + export function classifyValidateRuns(runs) { - if (!Array.isArray(runs) || runs.length === 0) return 'wait'; - if (runs.some((run) => run?.status === 'completed' && run?.conclusion !== 'success')) return 'fail'; - if (runs.some((run) => run?.status !== 'completed')) return 'wait'; - return 'success'; + const latest = latestRelevantRun(runs); + if (!latest) return 'wait'; + if (latest.status !== 'completed') return 'wait'; + return latest.conclusion === 'success' ? 'success' : 'fail'; } export const workflowRunsUrl = ({ repo, workflow, sha }) => ( @@ -37,14 +52,14 @@ export async function waitForGreenWorkflow({ const body = await response.json(); const runs = Array.isArray(body?.workflow_runs) ? body.workflow_runs : []; const state = classifyValidateRuns(runs); + const latest = latestRelevantRun(runs); if (state === 'fail') { - const failed = runs.filter((run) => run?.status === 'completed' && run?.conclusion !== 'success'); - throw new Error(`${label} is not green for ${sha}: ${JSON.stringify(failed.map((run) => ({ - conclusion: run.conclusion, url: run.html_url, - })))}`); + throw new Error(`${label} is not green for ${sha}: latest run ${JSON.stringify({ + conclusion: latest.conclusion, url: latest.html_url, + })}`); } if (state === 'success') { - console.log(`${label} is green for ${sha} (${runs.length} run(s))`); + console.log(`${label} is green for ${sha}: latest run ${latest.html_url || latest.id} (${runs.length} run(s) on the SHA)`); return; } if (Date.now() >= deadline) throw new Error(`No completed green ${label} for ${sha} within the deadline`); diff --git a/test/release-gate.test.mjs b/test/release-gate.test.mjs index 29435b6c..3714cba1 100644 --- a/test/release-gate.test.mjs +++ b/test/release-gate.test.mjs @@ -1,13 +1,14 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { classifyValidateRuns, workflowRunsUrl } from '../scripts/release-gate.mjs'; +import { readFileSync } from 'node:fs'; +import { classifyValidateRuns, latestRelevantRun, workflowRunsUrl } from '../scripts/release-gate.mjs'; test('release gate waits until an exact-SHA Validate exists and completes', () => { assert.equal(classifyValidateRuns([]), 'wait'); assert.equal(classifyValidateRuns([{ status: 'queued', conclusion: null }]), 'wait'); assert.equal(classifyValidateRuns([ - { status: 'completed', conclusion: 'success' }, - { status: 'in_progress', conclusion: null }, + { id: 1, run_started_at: '2026-09-09T13:01:00Z', status: 'completed', conclusion: 'success' }, + { id: 2, run_started_at: '2026-09-09T13:02:00Z', status: 'in_progress', conclusion: null }, ]), 'wait'); }); @@ -18,8 +19,8 @@ test('release gate accepts only completed success runs', () => { ]), 'success'); }); -test('release gate fails closed for red, cancelled and skipped runs', () => { - for (const conclusion of ['failure', 'cancelled', 'timed_out', 'action_required', 'skipped', null]) { +test('release gate fails closed for red, timed-out and skipped runs (cancelled ones are not verdicts, #511)', () => { + for (const conclusion of ['failure', 'timed_out', 'action_required', 'skipped', null]) { assert.equal( classifyValidateRuns([{ status: 'completed', conclusion }]), 'fail', @@ -28,11 +29,22 @@ test('release gate fails closed for red, cancelled and skipped runs', () => { } }); -test('one red duplicate blocks a green duplicate for the same SHA', () => { - assert.equal(classifyValidateRuns([ - { status: 'completed', conclusion: 'success' }, - { status: 'completed', conclusion: 'failure' }, - ]), 'fail'); +test('#511: the latest non-cancelled run is the verdict; cancelled runs prove nothing', () => { + const at = (minute, over) => ({ id: minute, run_started_at: `2026-09-09T13:${String(minute).padStart(2, '0')}:00Z`, status: 'completed', ...over }); + // an older green does not outrank a newer red … + assert.equal(classifyValidateRuns([at(1, { conclusion: 'success' }), at(2, { conclusion: 'failure' })]), 'fail'); + // … and a newer green (re-run, dispatch with another baseline) refreshes an older red + assert.equal(classifyValidateRuns([at(1, { conclusion: 'failure' }), at(2, { conclusion: 'success' })]), 'success'); + // order in the payload is irrelevant: the timestamp decides + assert.equal(classifyValidateRuns([at(2, { conclusion: 'success' }), at(1, { conclusion: 'failure' })]), 'success'); + // cancelled twins are invisible + assert.equal(classifyValidateRuns([at(1, { conclusion: 'cancelled' })]), 'wait'); + assert.equal(classifyValidateRuns([at(2, { conclusion: 'cancelled' }), at(1, { conclusion: 'success' })]), 'success'); + assert.equal(classifyValidateRuns([at(2, { conclusion: 'cancelled' }), at(1, { conclusion: 'failure' })]), 'fail'); + // a newer run still going means wait, even with an older green behind it + assert.equal(classifyValidateRuns([at(1, { conclusion: 'success' }), at(2, { status: 'in_progress', conclusion: null })]), 'wait'); + assert.equal(latestRelevantRun([at(2, { conclusion: 'cancelled' }), at(1, { conclusion: 'success' })]).id, 1); + assert.equal(latestRelevantRun([]), null); }); test('release gate can target the dedicated exact-SHA performance workflow', () => { @@ -41,3 +53,11 @@ test('release gate can target the dedicated exact-SHA performance workflow', () 'https://api.github.com/repos/Matysh/houseplan-card/actions/workflows/performance.yml/runs?head_sha=abc%2F123&per_page=100', ); }); + +test('#511 AC3: the release documents describe the latest-run semantics', () => { + const development = readFileSync(new URL('../docs/DEVELOPMENT.md', import.meta.url), 'utf8'); + assert.match(development, /latest non-cancelled Validate run of the\nSHA/); + assert.doesNotMatch(development, /A missing, failed,\ncancelled or one-hour-timed-out Validate withholds/); + const performance = readFileSync(new URL('../demo/performance/README.md', import.meta.url), 'utf8'); + assert.match(performance, /latest\nnon-cancelled run on the SHA/); +});