From 8d844f2d6127625a1bdfdc509da26dd0c0c7d54f Mon Sep 17 00:00:00 2001 From: Alex Dunmow Date: Tue, 7 Jul 2026 01:11:00 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20check=2030=20=E2=80=94=20render=20bench?= =?UTF-8?q?mark=20regression=20gate=20(WO-RP-013)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compares cms e2e/render-bench/results/latest.json (written by the dojo render-bench suite) against the committed baseline.json. FAILs when home-page p50 exceeds baseline by >25%, with an explicit instruction to investigate and never bump the baseline without Captain approval. p95 breach (>40%) WARNs only: measured ambient p95 variance on an idle dev instance exceeds 100% between runs while p50 holds within ~7%, so a hard tail gate would block unrelated commits on dev-stack noise. SKIPs (never blocks) when: baseline.json absent (non-cms repo), latest.json absent (bench not run), latest older than 7 days, or latest predates the baseline — each with a nudge to run the suite from dojo. Comparison logic is Reporter-free and unit-tested (renderperf_test.go). Verified against the live tree in all three states: OK (p50 22.9 vs 27.3ms), FAIL (26% slower via delay proxy, mandated message), SKIP (latest.json removed). Co-Authored-By: Claude Fable 5 --- check_renderperf.go | 51 ++++++++++++++++ renderperf.go | 138 ++++++++++++++++++++++++++++++++++++++++++++ renderperf_test.go | 76 ++++++++++++++++++++++++ 3 files changed, 265 insertions(+) create mode 100644 check_renderperf.go create mode 100644 renderperf.go create mode 100644 renderperf_test.go diff --git a/check_renderperf.go b/check_renderperf.go new file mode 100644 index 0000000..43f5c78 --- /dev/null +++ b/check_renderperf.go @@ -0,0 +1,51 @@ +package main + +import ( + "os" + "path/filepath" + "time" +) + +func init() { + register(Check{ + Seq: 300, + ID: "30", + Title: "Render benchmark: home-page p50 within baseline (WO-RP-013)", + Run: func(ctx *ScanContext, rep *Reporter) { + benchDir := filepath.Join(ctx.repoRoot, "e2e", "render-bench") + baselinePath := filepath.Join(benchDir, "baseline.json") + if _, err := os.Stat(baselinePath); err != nil { + rep.Skip("no e2e/render-bench/baseline.json in this repo — render-perf gate not applicable") + return + } + latestPath := filepath.Join(benchDir, "results", "latest.json") + if _, err := os.Stat(latestPath); err != nil { + rep.Skip("render bench not run (no results/latest.json) — run the `render-bench` suite from dojo to gate render performance") + return + } + + baseline, err := loadRenderBench(baselinePath) + if err != nil { + rep.Warn("render-perf baseline unreadable: %v", err) + return + } + latest, err := loadRenderBench(latestPath) + if err != nil { + rep.Warn("render-perf latest.json unreadable: %v — re-run the render-bench suite from dojo", err) + return + } + + v := compareRenderPerf(baseline, latest, time.Now()) + switch { + case v.skip: + rep.Skip("%s", v.message) + case v.fail: + rep.Fail("%s", v.message) + case v.warn: + rep.Warn("%s", v.message) + default: + rep.OK("%s", v.message) + } + }, + }) +} diff --git a/renderperf.go b/renderperf.go new file mode 100644 index 0000000..3bf0fc7 --- /dev/null +++ b/renderperf.go @@ -0,0 +1,138 @@ +package main + +// Render-performance regression gate (WO-RP-013). +// +// Compares the latest e2e render benchmark result (cms +// e2e/render-bench/results/latest.json, written by the dojo suite +// `render-bench`) against the committed baseline +// (e2e/render-bench/baseline.json) and FAILS when the home page — the gating +// page class — is markedly slower. +// +// Staleness rule: this check must never block unrelated work when no live +// stack is running. It SKIPs (never fails) when: +// - baseline.json is absent (repo is not the cms repo, or gate not set up), +// - results/latest.json is absent (bench not run on this machine), +// - latest.json is older than renderPerfMaxAge (the bench predates recent +// work and proves nothing about it), +// - latest.json predates the committed baseline (stale result from before +// the current baseline was captured). +// In every SKIP case it says how to produce a fresh result (run the +// `render-bench` suite from dojo). + +import ( + "encoding/json" + "fmt" + "os" + "time" +) + +const ( + // renderPerfP50Threshold fails the gate when home-page p50 exceeds the + // baseline p50 by more than this fraction. 25% sits above observed + // run-to-run variance (~7%) but well below any real regression worth a + // human look (WO-RP-013 mandated default). + renderPerfP50Threshold = 0.25 + // renderPerfP95Threshold is the secondary tail gate. It WARNs rather than + // FAILs: observed p95 on an otherwise-idle dev instance varies >100% + // between runs minutes apart (36ms → 76ms) while p50 stays within ~7%, + // so a hard p95 gate would block commits on ambient dev-stack noise. + // p50 is the blocking metric. + renderPerfP95Threshold = 0.40 + // renderPerfMaxAge is how old a latest.json may be and still count as + // evidence about the current tree. + renderPerfMaxAge = 7 * 24 * time.Hour +) + +type renderBenchPage struct { + Page string `json:"page"` + Path string `json:"path"` + Gating bool `json:"gating"` + P50Ms float64 `json:"p50_ms"` + P95Ms float64 `json:"p95_ms"` +} + +type renderBenchResults struct { + Timestamp string `json:"timestamp"` + Instance string `json:"instance"` + Pages []renderBenchPage `json:"pages"` +} + +func loadRenderBench(path string) (*renderBenchResults, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var r renderBenchResults + if err := json.Unmarshal(raw, &r); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + return &r, nil +} + +func (r *renderBenchResults) gatingPage() *renderBenchPage { + for i := range r.Pages { + if r.Pages[i].Gating { + return &r.Pages[i] + } + } + return nil +} + +func (r *renderBenchResults) parsedTime() (time.Time, error) { + return time.Parse(time.RFC3339, r.Timestamp) +} + +// renderPerfVerdict is the outcome of one baseline/latest comparison, +// separated from the Reporter so it can be unit-tested. +type renderPerfVerdict struct { + skip bool + warn bool + fail bool + message string +} + +func compareRenderPerf(baseline, latest *renderBenchResults, now time.Time) renderPerfVerdict { + base := baseline.gatingPage() + if base == nil { + return renderPerfVerdict{skip: true, message: "baseline.json has no gating page — regenerate it from the render-bench suite"} + } + cur := latest.gatingPage() + if cur == nil { + return renderPerfVerdict{skip: true, message: "latest.json has no gating page — re-run the render-bench suite from dojo"} + } + + latestAt, err := latest.parsedTime() + if err != nil { + return renderPerfVerdict{skip: true, message: fmt.Sprintf("latest.json has an unparseable timestamp (%v) — re-run the render-bench suite from dojo", err)} + } + if now.Sub(latestAt) > renderPerfMaxAge { + return renderPerfVerdict{skip: true, message: fmt.Sprintf("latest bench result is %.0fh old (max %.0fh) — run the render-bench suite from dojo for a fresh result", now.Sub(latestAt).Hours(), renderPerfMaxAge.Hours())} + } + if baseAt, err := baseline.parsedTime(); err == nil && latestAt.Before(baseAt) { + return renderPerfVerdict{skip: true, message: "latest bench result predates the committed baseline — run the render-bench suite from dojo"} + } + + if base.P50Ms <= 0 { + return renderPerfVerdict{skip: true, message: "baseline gating p50 is zero — regenerate baseline.json from the render-bench suite"} + } + + p50Delta := (cur.P50Ms - base.P50Ms) / base.P50Ms + p95Delta := 0.0 + if base.P95Ms > 0 { + p95Delta = (cur.P95Ms - base.P95Ms) / base.P95Ms + } + + if p50Delta > renderPerfP50Threshold { + return renderPerfVerdict{fail: true, message: fmt.Sprintf( + "Home page render is %.0f%% slower than baseline (p50 %.1fms vs %.1fms). Investigate before proceeding — profile the regression (see docs/works/WO-RP-001 methodology); do NOT raise the baseline to make this pass without Captain approval.", + p50Delta*100, cur.P50Ms, base.P50Ms)} + } + if p95Delta > renderPerfP95Threshold { + return renderPerfVerdict{warn: true, message: fmt.Sprintf( + "Home page render tail is %.0f%% slower than baseline (p95 %.1fms vs %.1fms; p50 healthy at %.1fms vs %.1fms). p95 is noisy on the shared dev stack — re-run the render-bench suite; investigate if it persists.", + p95Delta*100, cur.P95Ms, base.P95Ms, cur.P50Ms, base.P50Ms)} + } + + return renderPerfVerdict{message: fmt.Sprintf("home p50 %.1fms vs baseline %.1fms (%+.0f%%), p95 %.1fms vs %.1fms (%+.0f%%)", + cur.P50Ms, base.P50Ms, p50Delta*100, cur.P95Ms, base.P95Ms, p95Delta*100)} +} diff --git a/renderperf_test.go b/renderperf_test.go new file mode 100644 index 0000000..3ea6df5 --- /dev/null +++ b/renderperf_test.go @@ -0,0 +1,76 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +func benchFixture(ts string, p50, p95 float64) *renderBenchResults { + return &renderBenchResults{ + Timestamp: ts, + Instance: "blockninjacms.blockninja.dev", + Pages: []renderBenchPage{ + {Page: "home", Path: "/", Gating: true, P50Ms: p50, P95Ms: p95}, + {Page: "blog-post", Path: "/blog/x", P50Ms: 18, P95Ms: 25}, + }, + } +} + +func TestCompareRenderPerf(t *testing.T) { + now := time.Date(2026, 7, 7, 12, 0, 0, 0, time.UTC) + baseline := benchFixture("2026-07-06T16:47:00Z", 27.3, 37.0) + + t.Run("within threshold passes", func(t *testing.T) { + v := compareRenderPerf(baseline, benchFixture("2026-07-07T10:00:00Z", 30.0, 40.0), now) + if v.skip || v.fail { + t.Fatalf("expected pass, got %+v", v) + } + }) + + t.Run("p50 regression fails with investigate message", func(t *testing.T) { + v := compareRenderPerf(baseline, benchFixture("2026-07-07T10:00:00Z", 40.0, 41.0), now) + if !v.fail { + t.Fatalf("expected fail, got %+v", v) + } + for _, want := range []string{"slower than baseline", "Investigate before proceeding", "do NOT raise the baseline", "Captain approval"} { + if !strings.Contains(v.message, want) { + t.Errorf("fail message missing %q: %s", want, v.message) + } + } + }) + + t.Run("p95-only regression warns via secondary gate", func(t *testing.T) { + v := compareRenderPerf(baseline, benchFixture("2026-07-07T10:00:00Z", 28.0, 60.0), now) + if !v.warn || v.fail { + t.Fatalf("expected warn (not fail) for p95-only breach, got %+v", v) + } + if !strings.Contains(v.message, "p95") { + t.Errorf("expected p95 metric in message: %s", v.message) + } + }) + + t.Run("stale latest skips", func(t *testing.T) { + v := compareRenderPerf(baseline, benchFixture("2026-06-20T10:00:00Z", 40.0, 60.0), now) + if !v.skip { + t.Fatalf("expected skip for stale result, got %+v", v) + } + if !strings.Contains(v.message, "dojo") { + t.Errorf("skip message should say how to refresh: %s", v.message) + } + }) + + t.Run("latest predating baseline skips", func(t *testing.T) { + v := compareRenderPerf(baseline, benchFixture("2026-07-06T10:00:00Z", 40.0, 60.0), now) + if !v.skip { + t.Fatalf("expected skip for pre-baseline result, got %+v", v) + } + }) + + t.Run("boundary: exactly 25 percent passes", func(t *testing.T) { + v := compareRenderPerf(baseline, benchFixture("2026-07-07T10:00:00Z", 27.3*1.25, 37.0), now) + if v.fail { + t.Fatalf("exactly-threshold should not fail: %+v", v) + } + }) +}