diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 863d5b3d..599648ba 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -24,10 +24,20 @@ concurrency: jobs: performance: - # Base and candidate stay sequential on one hosted runner. Splitting them - # across runners would turn machine variance into a false regression. + # Every profile keeps base and candidate sequential on one hosted runner. + # Independent profile pairs run in parallel: cross-profile timing is never + # compared, while serialising all five pairs cannot fit the job timeout. runs-on: ubuntu-latest timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + profile: + - large-house + - isometric + - plan-snap + - blend + - overlay steps: - name: Check out candidate uses: actions/checkout@v7 @@ -158,36 +168,69 @@ jobs: cp baseline/dist/houseplan-card.js baseline/demo/srv/assets/houseplan-card.js fi - - name: Capture base and candidate profiles + - name: Capture base and candidate profile working-directory: candidate + env: + PROFILE: ${{ matrix.profile }} run: | - npm run benchmark:large-house -- --target-root=../baseline --samples=7 --warmups=1 --output=../artifacts/performance/baseline.json - npm run benchmark:large-house -- --target-root=. --samples=7 --warmups=1 --output=../artifacts/performance/candidate.json - npm run benchmark:large-house-isometric -- --target-root=../baseline --samples=7 --warmups=1 --output=../artifacts/performance/isometric-baseline.json - npm run benchmark:large-house-isometric -- --target-root=. --samples=7 --warmups=1 --output=../artifacts/performance/isometric-candidate.json - npm run benchmark:large-house-plan-snap -- --target-root=../baseline --samples=7 --warmups=1 --output=../artifacts/performance/plan-snap-baseline.json - npm run benchmark:large-house-plan-snap -- --target-root=. --samples=7 --warmups=1 --output=../artifacts/performance/plan-snap-candidate.json - npm run benchmark:glow -- --profile=large-light-blend-v1 --target-root=../baseline --samples=7 --warmups=1 --output=../artifacts/performance/blend-baseline.json - npm run benchmark:glow -- --profile=large-light-blend-v1 --target-root=. --samples=7 --warmups=1 --output=../artifacts/performance/blend-candidate.json - npm run benchmark:glow -- --profile=large-house-glow-overlay-v1 --target-root=../baseline --samples=7 --warmups=1 --output=../artifacts/performance/overlay-baseline.json - npm run benchmark:glow -- --profile=large-house-glow-overlay-v1 --target-root=. --samples=7 --warmups=1 --output=../artifacts/performance/overlay-candidate.json - if ! grep -q "glow_enabled" ../baseline/src/logic.ts; then - echo "Base predates independent Glow; bootstrap relative overlay baseline, keep absolute gate" - cp ../artifacts/performance/overlay-candidate.json ../artifacts/performance/overlay-baseline.json - fi + mkdir -p ../artifacts/performance + case "$PROFILE" in + large-house) + npm run benchmark:large-house -- --target-root=../baseline --samples=7 --warmups=1 --output=../artifacts/performance/baseline.json + npm run benchmark:large-house -- --target-root=. --samples=7 --warmups=1 --output=../artifacts/performance/candidate.json + ;; + isometric) + npm run benchmark:large-house-isometric -- --target-root=../baseline --samples=7 --warmups=1 --output=../artifacts/performance/isometric-baseline.json + npm run benchmark:large-house-isometric -- --target-root=. --samples=7 --warmups=1 --output=../artifacts/performance/isometric-candidate.json + ;; + plan-snap) + npm run benchmark:large-house-plan-snap -- --target-root=../baseline --samples=7 --warmups=1 --output=../artifacts/performance/plan-snap-baseline.json + npm run benchmark:large-house-plan-snap -- --target-root=. --samples=7 --warmups=1 --output=../artifacts/performance/plan-snap-candidate.json + ;; + blend) + npm run benchmark:glow -- --profile=large-light-blend-v1 --target-root=../baseline --samples=7 --warmups=1 --output=../artifacts/performance/blend-baseline.json + npm run benchmark:glow -- --profile=large-light-blend-v1 --target-root=. --samples=7 --warmups=1 --output=../artifacts/performance/blend-candidate.json + ;; + overlay) + npm run benchmark:glow -- --profile=large-house-glow-overlay-v1 --target-root=../baseline --samples=7 --warmups=1 --output=../artifacts/performance/overlay-baseline.json + npm run benchmark:glow -- --profile=large-house-glow-overlay-v1 --target-root=. --samples=7 --warmups=1 --output=../artifacts/performance/overlay-candidate.json + if ! grep -q "glow_enabled" ../baseline/src/logic.ts; then + echo "Base predates independent Glow; bootstrap relative overlay baseline, keep absolute gate" + cp ../artifacts/performance/overlay-candidate.json ../artifacts/performance/overlay-baseline.json + fi + ;; + *) + echo "::error::Unknown performance profile: $PROFILE" + exit 1 + ;; + esac - - name: Enforce relative and absolute performance budgets + - name: Enforce relative and absolute performance budget working-directory: candidate + env: + PROFILE: ${{ matrix.profile }} run: | - npm run benchmark:compare -- --baseline=../artifacts/performance/baseline.json --candidate=../artifacts/performance/candidate.json --output=../artifacts/performance/comparison.json - npm run benchmark:compare -- --budgets=demo/performance/budgets-large-house-isometric.json --baseline=../artifacts/performance/isometric-baseline.json --candidate=../artifacts/performance/isometric-candidate.json --output=../artifacts/performance/isometric-comparison.json - npm run benchmark:compare -- --budgets=demo/performance/budgets-large-house-plan-snap.json --baseline=../artifacts/performance/plan-snap-baseline.json --candidate=../artifacts/performance/plan-snap-candidate.json --output=../artifacts/performance/plan-snap-comparison.json - npm run benchmark:compare -- --budgets=demo/performance/budgets-large-light-blend.json --baseline=../artifacts/performance/blend-baseline.json --candidate=../artifacts/performance/blend-candidate.json --output=../artifacts/performance/blend-comparison.json - npm run benchmark:compare -- --budgets=demo/performance/budgets-large-house-glow-overlay.json --baseline=../artifacts/performance/overlay-baseline.json --candidate=../artifacts/performance/overlay-candidate.json --output=../artifacts/performance/overlay-comparison.json + case "$PROFILE" in + large-house) + npm run benchmark:compare -- --baseline=../artifacts/performance/baseline.json --candidate=../artifacts/performance/candidate.json --output=../artifacts/performance/comparison.json + ;; + isometric) + npm run benchmark:compare -- --budgets=demo/performance/budgets-large-house-isometric.json --baseline=../artifacts/performance/isometric-baseline.json --candidate=../artifacts/performance/isometric-candidate.json --output=../artifacts/performance/isometric-comparison.json + ;; + plan-snap) + npm run benchmark:compare -- --budgets=demo/performance/budgets-large-house-plan-snap.json --baseline=../artifacts/performance/plan-snap-baseline.json --candidate=../artifacts/performance/plan-snap-candidate.json --output=../artifacts/performance/plan-snap-comparison.json + ;; + blend) + npm run benchmark:compare -- --budgets=demo/performance/budgets-large-light-blend.json --baseline=../artifacts/performance/blend-baseline.json --candidate=../artifacts/performance/blend-candidate.json --output=../artifacts/performance/blend-comparison.json + ;; + overlay) + npm run benchmark:compare -- --budgets=demo/performance/budgets-large-house-glow-overlay.json --baseline=../artifacts/performance/overlay-baseline.json --candidate=../artifacts/performance/overlay-candidate.json --output=../artifacts/performance/overlay-comparison.json + ;; + esac - - name: Upload full performance reports + - name: Upload full performance report if: always() uses: actions/upload-artifact@v7 with: - name: full-performance + name: full-performance-${{ matrix.profile }} path: artifacts/performance diff --git a/demo/performance/README.md b/demo/performance/README.md index d3c0fd6a..33d0c562 100644 --- a/demo/performance/README.md +++ b/demo/performance/README.md @@ -65,9 +65,11 @@ regressions. Small fast operations receive an absolute noise allowance so normal scheduler jitter does not become a false regression. Heap, Long Tasks, warmed-cache growth and the expected rendered-device count are gated separately. Long-Task maximum/count/total checks use the same -relative-plus-absolute policy as timings. Both raw reports and the comparison -are always uploaded as the `full-performance` artifact, and the table is -written to the GitHub job summary. Stable release assets require both exact-SHA +relative-plus-absolute policy as timings. Each independent profile pair runs in +parallel with the other pairs, but its base and candidate remain sequential on +one runner. Its raw reports and comparison are uploaded as +`full-performance-`, and the table is written to that GitHub job +summary. Stable release assets require both exact-SHA `Validate` and exact-SHA `Full Performance`; prereleases require only `Validate`. diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 6094fc1f..249d0504 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -147,10 +147,13 @@ npm run golden:accept -- --reviewed The config audit performs no network requests and does not rewrite the input. Its registry and lifecycle rules are documented in `CONFIG-COMPATIBILITY.md`. -The blocking performance job captures the base SHA and candidate sequentially -on one pinned Chromium/CI profile, applies relative and absolute budgets, and -uploads both reports plus the comparison. A developer-laptop report remains a -diagnostic and must not be used to loosen CI limits. See +The blocking performance workflow runs its independent profile pairs in +parallel. Inside each pair it captures the base SHA and candidate sequentially +on one pinned Chromium/CI runner, applies the same relative and absolute budget, +and uploads both reports plus the comparison. This preserves same-machine +comparability without serialising the whole matrix beyond the job timeout. A +developer-laptop report remains a diagnostic and must not be used to loosen CI +limits. See `demo/performance/README.md`. When the comparison base predates `scripts/bundle-sync.mjs`, the workflow still builds that exact tree and materializes its fresh bundle through the equivalent diff --git a/test/performance-workflow.test.mjs b/test/performance-workflow.test.mjs index 7867f3ff..5021cf39 100644 --- a/test/performance-workflow.test.mjs +++ b/test/performance-workflow.test.mjs @@ -27,10 +27,21 @@ test('full performance is isolated to stable, scheduled and manual entry points' '- main', 'schedule:', 'workflow_dispatch:', - 'Capture base and candidate profiles', + 'Capture base and candidate profile', + 'profile:', + '- large-house', + '- isometric', + '- plan-snap', + '- blend', + '- overlay', + 'PROFILE: ${{ matrix.profile }}', + 'name: full-performance-${{ matrix.profile }}', '--samples=7 --warmups=1', ]) assert.ok(workflow.includes(contract), `missing full-gate contract: ${contract}`); + assert.ok(workflow.includes('if [ -f baseline/scripts/bundle-sync.mjs ]; then')); + assert.equal((workflow.match(/--samples=7 --warmups=1/g) || []).length, 10); + const release = readWorkflow('release.yml'); assert.ok(release.includes('if: ${{ !github.event.release.prerelease }}')); assert.ok(release.includes('--workflow=performance.yml --label="Full Performance"'));