block/check-safety
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: check 30 — render benchmark regression gate (WO-RP-013)

Browse Source 
Compares cms e2e/render-bench/results/latest.json (written by the dojo
render-bench suite) against the committed baseline.json. FAILs when
home-page p50 exceeds baseline by >25%, with an explicit instruction to
investigate and never bump the baseline without Captain approval. p95
breach (>40%) WARNs only: measured ambient p95 variance on an idle dev
instance exceeds 100% between runs while p50 holds within ~7%, so a hard
tail gate would block unrelated commits on dev-stack noise.

SKIPs (never blocks) when: baseline.json absent (non-cms repo),
latest.json absent (bench not run), latest older than 7 days, or latest
predates the baseline — each with a nudge to run the suite from dojo.
Comparison logic is Reporter-free and unit-tested (renderperf_test.go).

Verified against the live tree in all three states: OK (p50 22.9 vs
27.3ms), FAIL (26% slower via delay proxy, mandated message), SKIP
(latest.json removed).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Alex Dunmow 2026-07-07 01:11:00 +08:00
parent 422674ded0
commit 8d844f2d61
3 changed files with 265 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
check_renderperf.go Normal file
View File

@ -0,0 +1,51 @@
package main
import (
"os"
"path/filepath"
"time"
)
func init() {
register(Check{
Seq: 300,
ID: "30",
Title: "Render benchmark: home-page p50 within baseline (WO-RP-013)",
Run: func(ctx *ScanContext, rep *Reporter) {
benchDir := filepath.Join(ctx.repoRoot, "e2e", "render-bench")
baselinePath := filepath.Join(benchDir, "baseline.json")
if _, err := os.Stat(baselinePath); err != nil {
rep.Skip("no e2e/render-bench/baseline.json in this repo — render-perf gate not applicable")
return
}
latestPath := filepath.Join(benchDir, "results", "latest.json")
if _, err := os.Stat(latestPath); err != nil {
rep.Skip("render bench not run (no results/latest.json) — run the `render-bench` suite from dojo to gate render performance")
return
}
baseline, err := loadRenderBench(baselinePath)
if err != nil {
rep.Warn("render-perf baseline unreadable: %v", err)
return
}
latest, err := loadRenderBench(latestPath)
if err != nil {
rep.Warn("render-perf latest.json unreadable: %v — re-run the render-bench suite from dojo", err)
return
}
v := compareRenderPerf(baseline, latest, time.Now())
switch {
case v.skip:
rep.Skip("%s", v.message)
case v.fail:
rep.Fail("%s", v.message)
case v.warn:
rep.Warn("%s", v.message)
default:
rep.OK("%s", v.message)
}
},
})
}

138
renderperf.go Normal file
View File

@ -0,0 +1,138 @@
package main
// Render-performance regression gate (WO-RP-013).
//
// Compares the latest e2e render benchmark result (cms
// e2e/render-bench/results/latest.json, written by the dojo suite
// `render-bench`) against the committed baseline
// (e2e/render-bench/baseline.json) and FAILS when the home page — the gating
// page class — is markedly slower.
//
// Staleness rule: this check must never block unrelated work when no live
// stack is running. It SKIPs (never fails) when:
// - baseline.json is absent (repo is not the cms repo, or gate not set up),
// - results/latest.json is absent (bench not run on this machine),
// - latest.json is older than renderPerfMaxAge (the bench predates recent
// work and proves nothing about it),
// - latest.json predates the committed baseline (stale result from before
// the current baseline was captured).
// In every SKIP case it says how to produce a fresh result (run the
// `render-bench` suite from dojo).
import (
"encoding/json"
"fmt"
"os"
"time"
)
const (
// renderPerfP50Threshold fails the gate when home-page p50 exceeds the
// baseline p50 by more than this fraction. 25% sits above observed
// run-to-run variance (~7%) but well below any real regression worth a
// human look (WO-RP-013 mandated default).
renderPerfP50Threshold = 0.25
// renderPerfP95Threshold is the secondary tail gate. It WARNs rather than
// FAILs: observed p95 on an otherwise-idle dev instance varies >100%
// between runs minutes apart (36ms → 76ms) while p50 stays within ~7%,
// so a hard p95 gate would block commits on ambient dev-stack noise.
// p50 is the blocking metric.
renderPerfP95Threshold = 0.40
// renderPerfMaxAge is how old a latest.json may be and still count as
// evidence about the current tree.
renderPerfMaxAge = 7 * 24 * time.Hour
)
type renderBenchPage struct {
Page string `json:"page"`
Path string `json:"path"`
Gating bool `json:"gating"`
P50Ms float64 `json:"p50_ms"`
P95Ms float64 `json:"p95_ms"`
}
type renderBenchResults struct {
Timestamp string `json:"timestamp"`
Instance string `json:"instance"`
Pages []renderBenchPage `json:"pages"`
}
func loadRenderBench(path string) (*renderBenchResults, error) {
raw, err := os.ReadFile(path)
if err != nil {
return nil, err
}
var r renderBenchResults
if err := json.Unmarshal(raw, &r); err != nil {
return nil, fmt.Errorf("%s: %w", path, err)
}
return &r, nil
}
func (r *renderBenchResults) gatingPage() *renderBenchPage {
for i := range r.Pages {
if r.Pages[i].Gating {
return &r.Pages[i]
}
}
return nil
}
func (r *renderBenchResults) parsedTime() (time.Time, error) {
return time.Parse(time.RFC3339, r.Timestamp)
}
// renderPerfVerdict is the outcome of one baseline/latest comparison,
// separated from the Reporter so it can be unit-tested.
type renderPerfVerdict struct {
skip bool
warn bool
fail bool
message string
}
func compareRenderPerf(baseline, latest *renderBenchResults, now time.Time) renderPerfVerdict {
base := baseline.gatingPage()
if base == nil {
return renderPerfVerdict{skip: true, message: "baseline.json has no gating page — regenerate it from the render-bench suite"}
}
cur := latest.gatingPage()
if cur == nil {
return renderPerfVerdict{skip: true, message: "latest.json has no gating page — re-run the render-bench suite from dojo"}
}
latestAt, err := latest.parsedTime()
if err != nil {
return renderPerfVerdict{skip: true, message: fmt.Sprintf("latest.json has an unparseable timestamp (%v) — re-run the render-bench suite from dojo", err)}
}
if now.Sub(latestAt) > renderPerfMaxAge {
return renderPerfVerdict{skip: true, message: fmt.Sprintf("latest bench result is %.0fh old (max %.0fh) — run the render-bench suite from dojo for a fresh result", now.Sub(latestAt).Hours(), renderPerfMaxAge.Hours())}
}
if baseAt, err := baseline.parsedTime(); err == nil && latestAt.Before(baseAt) {
return renderPerfVerdict{skip: true, message: "latest bench result predates the committed baseline — run the render-bench suite from dojo"}
}
if base.P50Ms <= 0 {
return renderPerfVerdict{skip: true, message: "baseline gating p50 is zero — regenerate baseline.json from the render-bench suite"}
}
p50Delta := (cur.P50Ms - base.P50Ms) / base.P50Ms
p95Delta := 0.0
if base.P95Ms > 0 {
p95Delta = (cur.P95Ms - base.P95Ms) / base.P95Ms
}
if p50Delta > renderPerfP50Threshold {
return renderPerfVerdict{fail: true, message: fmt.Sprintf(
"Home page render is %.0f%% slower than baseline (p50 %.1fms vs %.1fms). Investigate before proceeding — profile the regression (see docs/works/WO-RP-001 methodology); do NOT raise the baseline to make this pass without Captain approval.",
p50Delta*100, cur.P50Ms, base.P50Ms)}
}
if p95Delta > renderPerfP95Threshold {
return renderPerfVerdict{warn: true, message: fmt.Sprintf(
"Home page render tail is %.0f%% slower than baseline (p95 %.1fms vs %.1fms; p50 healthy at %.1fms vs %.1fms). p95 is noisy on the shared dev stack — re-run the render-bench suite; investigate if it persists.",
p95Delta*100, cur.P95Ms, base.P95Ms, cur.P50Ms, base.P50Ms)}
}
return renderPerfVerdict{message: fmt.Sprintf("home p50 %.1fms vs baseline %.1fms (%+.0f%%), p95 %.1fms vs %.1fms (%+.0f%%)",
cur.P50Ms, base.P50Ms, p50Delta*100, cur.P95Ms, base.P95Ms, p95Delta*100)}
}

76
renderperf_test.go Normal file
View File

@ -0,0 +1,76 @@
package main
import (
"strings"
"testing"
"time"
)
func benchFixture(ts string, p50, p95 float64) *renderBenchResults {
return &renderBenchResults{
Timestamp: ts,
Instance: "blockninjacms.blockninja.dev",
Pages: []renderBenchPage{
{Page: "home", Path: "/", Gating: true, P50Ms: p50, P95Ms: p95},
{Page: "blog-post", Path: "/blog/x", P50Ms: 18, P95Ms: 25},
},
}
}
func TestCompareRenderPerf(t *testing.T) {
now := time.Date(2026, 7, 7, 12, 0, 0, 0, time.UTC)
baseline := benchFixture("2026-07-06T16:47:00Z", 27.3, 37.0)
t.Run("within threshold passes", func(t *testing.T) {
v := compareRenderPerf(baseline, benchFixture("2026-07-07T10:00:00Z", 30.0, 40.0), now)
if v.skip || v.fail {
t.Fatalf("expected pass, got %+v", v)
}
})
t.Run("p50 regression fails with investigate message", func(t *testing.T) {
v := compareRenderPerf(baseline, benchFixture("2026-07-07T10:00:00Z", 40.0, 41.0), now)
if !v.fail {
t.Fatalf("expected fail, got %+v", v)
}
for _, want := range []string{"slower than baseline", "Investigate before proceeding", "do NOT raise the baseline", "Captain approval"} {
if !strings.Contains(v.message, want) {
t.Errorf("fail message missing %q: %s", want, v.message)
}
}
})
t.Run("p95-only regression warns via secondary gate", func(t *testing.T) {
v := compareRenderPerf(baseline, benchFixture("2026-07-07T10:00:00Z", 28.0, 60.0), now)
if !v.warn || v.fail {
t.Fatalf("expected warn (not fail) for p95-only breach, got %+v", v)
}
if !strings.Contains(v.message, "p95") {
t.Errorf("expected p95 metric in message: %s", v.message)
}
})
t.Run("stale latest skips", func(t *testing.T) {
v := compareRenderPerf(baseline, benchFixture("2026-06-20T10:00:00Z", 40.0, 60.0), now)
if !v.skip {
t.Fatalf("expected skip for stale result, got %+v", v)
}
if !strings.Contains(v.message, "dojo") {
t.Errorf("skip message should say how to refresh: %s", v.message)
}
})
t.Run("latest predating baseline skips", func(t *testing.T) {
v := compareRenderPerf(baseline, benchFixture("2026-07-06T10:00:00Z", 40.0, 60.0), now)
if !v.skip {
t.Fatalf("expected skip for pre-baseline result, got %+v", v)
}
})
t.Run("boundary: exactly 25 percent passes", func(t *testing.T) {
v := compareRenderPerf(baseline, benchFixture("2026-07-07T10:00:00Z", 27.3*1.25, 37.0), now)
if v.fail {
t.Fatalf("exactly-threshold should not fail: %+v", v)
}
})
}