const percentile = (values, p) => { if (!values.length) return null; const sorted = [...values].sort((a, b) => a - b); return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(sorted.length * p) - 1))]; }; const finite = (value) => typeof value === 'number' && Number.isFinite(value); const round = (value, digits = 2) => finite(value) ? Number(value.toFixed(digits)) : value; export const summarizeTimings = (rows, metricNames) => Object.fromEntries( metricNames.map((metric) => { const values = rows.map((row) => row[metric]).filter(finite); return [metric, { median: percentile(values, 0.5), p95: percentile(values, 0.95), min: values.length ? Math.min(...values) : null, max: values.length ? Math.max(...values) : null, }]; }), ); export const summarizeLongTasks = (rows) => { const totals = []; const counts = []; const singles = []; for (const row of rows) { const windows = Object.values(row.longTasks ?? {}); totals.push(windows.reduce((sum, item) => sum + (item?.totalMs ?? 0), 0)); counts.push(windows.reduce((sum, item) => sum + (item?.count ?? 0), 0)); singles.push(...windows.map((item) => item?.maxMs ?? 0)); } return { maxSingleMs: round(singles.length ? Math.max(...singles) : 0), countP95: percentile(counts, 0.95) ?? 0, totalP95Ms: round(percentile(totals, 0.95) ?? 0), }; }; const sameJson = (a, b) => JSON.stringify(a) === JSON.stringify(b); const exactGitSha = (value) => typeof value === 'string' && /^[0-9a-f]{40}$/i.test(value); const isIsometricProfile = (profile) => profile === 'large-house-isometric-v1' || profile === 'isometric-stage3-dense-v1'; const requireReport = (report, budgets, label, expectedSha = null) => { if (!report || report.schema !== 2) throw new Error(`${label}: unsupported report schema`); if (report.profile !== budgets.profile) throw new Error(`${label}: unexpected benchmark profile`); if (!Array.isArray(report.rows) || report.rows.length < budgets.minimumSamples) { throw new Error(`${label}: expected at least ${budgets.minimumSamples} measured samples`); } if (!isIsometricProfile(report.profile)) return; if (!exactGitSha(report.sourceSha)) throw new Error(`${label}: missing exact sourceSha`); if (expectedSha != null && report.sourceSha !== expectedSha) { throw new Error(`${label}: sourceSha does not match requested checkout`); } if (!sameJson(report.effectiveProjection, ['iso']) || report.rows.some((row) => row.effectiveProjection !== 'iso')) { throw new Error(`${label}: effectiveProjection is not exclusively iso`); } if (label === 'candidate' && report.rows.some((row) => row.isoStructuralBuilds?.supported !== true || row.isoStructuralBuilds?.haUpdateDelta !== 0)) { throw new Error(`${label}: Iso structural build count is missing or changed on HA-only update`); } if (label === 'candidate' && report.profile === 'isometric-stage3-dense-v1') { // The profile/schema name is intentionally stable for baseline continuity; // the candidate DOM revision tracks the current experimental Stage 4. if (report.stage3Required !== true) throw new Error(`${label}: Stage 4 was not required`); if (!sameJson(report.isoStageRevision, ['4']) || report.rows.some((row) => row.isoStageRevision !== '4')) { throw new Error(`${label}: Stage 4 revision metadata is incomplete`); } } }; const relativeLimit = (baseline, ratio, allowance) => Math.max( baseline * (1 + ratio), baseline + allowance, ); const makeCheck = (id, actual, limit, details = {}) => ({ id, actual: round(actual), limit: round(limit), pass: finite(actual) && actual <= limit, ...details, }); /** * Evaluate a candidate report against stable absolute ceilings. Full captures * additionally compare a base-SHA report from the same runner; fast smoke * captures deliberately enforce only the hard candidate limits. */ export const evaluatePerformanceBudget = ({ candidate, baseline, budgets, absoluteOnly = false, candidateSha = null, baselineSha = null, }) => { requireReport(candidate, budgets, 'candidate', candidateSha); if (!absoluteOnly) { requireReport(baseline, budgets, 'baseline', baselineSha); if (!sameJson(candidate.fixture, baseline.fixture)) throw new Error('fixture mismatch'); for (const key of ['node', 'chromium', 'platform', 'arch']) { if (candidate.runtime?.[key] !== baseline.runtime?.[key]) { throw new Error(`runtime mismatch for ${key}`); } } } const checks = []; for (const [metric, budget] of Object.entries(budgets.timings)) { const stat = budget.stat ?? 'median'; const actual = candidate.summary?.[metric]?.[stat]; const base = absoluteOnly ? null : baseline.summary?.[metric]?.[stat]; if (!finite(actual) || (!absoluteOnly && !finite(base))) throw new Error(`missing ${stat} for ${metric}`); const regressionLimit = absoluteOnly ? Number.POSITIVE_INFINITY : relativeLimit(base, budget.maxRegressionRatio, budget.noiseAllowanceMs); checks.push(makeCheck( `timing.${metric}.${stat}`, actual, Math.min(budget.hardMaxMs, regressionLimit), { ...(absoluteOnly ? {} : { baseline: round(base), regressionLimit: round(regressionLimit) }), hardLimit: budget.hardMaxMs, }, )); } const candidateLong = candidate.longTasks ?? summarizeLongTasks(candidate.rows); const baselineLong = absoluteOnly ? null : (baseline.longTasks ?? summarizeLongTasks(baseline.rows)); const longTasksAvailable = candidate.rows.every((row) => { const windows = Object.values(row.longTasks ?? {}); return windows.length > 0 && windows.every((item) => item?.supported === true); }); checks.push({ id: 'longTask.available', actual: longTasksAvailable ? 1 : 0, limit: 1, pass: longTasksAvailable }); const singleRegressionLimit = absoluteOnly ? Number.POSITIVE_INFINITY : relativeLimit( baselineLong.maxSingleMs, budgets.longTasks.maxSingleRegressionRatio, budgets.longTasks.maxSingleNoiseAllowanceMs, ); checks.push(makeCheck( 'longTask.maxSingleMs', candidateLong.maxSingleMs, Math.min(budgets.longTasks.maxSingleMs, singleRegressionLimit), { ...(absoluteOnly ? {} : { baseline: baselineLong.maxSingleMs }), hardLimit: budgets.longTasks.maxSingleMs, }, )); const countRegressionLimit = absoluteOnly ? Number.POSITIVE_INFINITY : relativeLimit( baselineLong.countP95, budgets.longTasks.maxCountRegressionRatio, budgets.longTasks.countNoiseAllowance, ); checks.push(makeCheck( 'longTask.countP95', candidateLong.countP95, Math.min(budgets.longTasks.maxCountP95, countRegressionLimit), { ...(absoluteOnly ? {} : { baseline: baselineLong.countP95 }), hardLimit: budgets.longTasks.maxCountP95, }, )); const longRegressionLimit = absoluteOnly ? Number.POSITIVE_INFINITY : relativeLimit( baselineLong.totalP95Ms, budgets.longTasks.maxTotalRegressionRatio, budgets.longTasks.noiseAllowanceMs, ); checks.push(makeCheck( 'longTask.totalP95Ms', candidateLong.totalP95Ms, Math.min(budgets.longTasks.maxTotalP95Ms, longRegressionLimit), { ...(absoluteOnly ? {} : { baseline: baselineLong.totalP95Ms }), hardLimit: budgets.longTasks.maxTotalP95Ms, }, )); for (const [windowName, budget] of Object.entries(budgets.longTaskWindows ?? {})) { const windows = candidate.rows.map((row) => row.longTasks?.[windowName]); const available = windows.every((window) => window?.supported === true); checks.push({ id: `longTask.${windowName}.available`, actual: available ? 1 : 0, limit: 1, pass: available, }); const maxSingle = available ? Math.max(...windows.map((window) => window.maxMs)) : NaN; const countP95 = available ? percentile(windows.map((window) => window.count), 0.95) : NaN; const totalP95 = available ? percentile(windows.map((window) => window.totalMs), 0.95) : NaN; checks.push(makeCheck(`longTask.${windowName}.maxSingleMs`, maxSingle, budget.maxSingleMs)); checks.push(makeCheck(`longTask.${windowName}.countP95`, countP95, budget.maxCountP95)); checks.push(makeCheck(`longTask.${windowName}.totalP95Ms`, totalP95, budget.maxTotalP95Ms)); } const candidateHeap = candidate.rows .map((row) => row.heapGrowthBytes) .filter(finite) .map((value) => Math.max(0, value)); const baselineHeap = absoluteOnly ? [] : baseline.rows .map((row) => row.heapGrowthBytes) .filter(finite) .map((value) => Math.max(0, value)); const preciseGc = candidate.rows.every((row) => row.preciseGc === true); checks.push({ id: 'heap.preciseGc', actual: preciseGc ? 1 : 0, limit: budgets.heap.required ? 1 : 0, pass: !budgets.heap.required || preciseGc, }); if (budgets.heap.required && (!candidateHeap.length || (!absoluteOnly && !baselineHeap.length))) { checks.push({ id: 'heap.available', actual: candidateHeap.length, limit: 1, pass: false }); } else if (candidateHeap.length && (absoluteOnly || baselineHeap.length)) { const actual = percentile(candidateHeap, 0.95); const base = absoluteOnly ? null : percentile(baselineHeap, 0.95); const regressionLimit = absoluteOnly ? Number.POSITIVE_INFINITY : relativeLimit( base, budgets.heap.maxRegressionRatio, budgets.heap.noiseAllowanceBytes, ); checks.push(makeCheck( 'heap.growthP95Bytes', actual, Math.min(budgets.heap.hardMaxGrowthBytes, regressionLimit), { ...(absoluteOnly ? {} : { baseline: base }), hardLimit: budgets.heap.hardMaxGrowthBytes, }, )); } for (const [cache, limit] of Object.entries(budgets.cacheEntries)) { const actual = Math.max(...candidate.rows.map((row) => row.cacheEntries?.[cache] ?? Number.POSITIVE_INFINITY)); checks.push(makeCheck(`cache.entries.${cache}`, actual, limit)); } for (const [cache, limit] of Object.entries(budgets.cacheGrowth)) { const actual = Math.max(...candidate.rows.map((row) => row.cacheGrowth?.[cache] ?? Number.POSITIVE_INFINITY)); checks.push(makeCheck(`cache.growth.${cache}`, actual, limit)); } const renderedDevices = Math.min(...candidate.rows.map((row) => row.renderedDevices ?? -1)); checks.push({ id: 'renderedDevices', actual: renderedDevices, limit: budgets.renderedDevices, pass: renderedDevices === budgets.renderedDevices, }); const failures = checks.filter((check) => !check.pass); return { schema: 1, profile: budgets.profile, pass: failures.length === 0, mode: absoluteOnly ? 'absolute' : 'relative', candidateFingerprint: candidate.buildFingerprint, baselineFingerprint: baseline?.buildFingerprint ?? null, candidateSha: candidate.sourceSha ?? null, baselineSha: baseline?.sourceSha ?? null, checks, failures, }; }; export const performanceSummaryMarkdown = (evaluation) => { const icon = evaluation.pass ? '✅' : '❌'; const lines = [ `### ${icon} House Plan large-house performance`, '', '| Check | Candidate | Limit | Base |', '|---|---:|---:|---:|', ]; for (const check of evaluation.checks) { lines.push(`| ${check.pass ? '✅' : '❌'} ${check.id} | ${check.actual} | ${check.limit} | ${check.baseline ?? '—'} |`); } return `${lines.join('\n')}\n`; };