package main // Render-performance regression gate (WO-RP-013). // // Compares the latest e2e render benchmark result (cms // e2e/render-bench/results/latest.json, written by the dojo suite // `render-bench`) against the committed baseline // (e2e/render-bench/baseline.json) and FAILS when the home page — the gating // page class — is markedly slower. // // Staleness rule: this check must never block unrelated work when no live // stack is running. It SKIPs (never fails) when: // - baseline.json is absent (repo is not the cms repo, or gate not set up), // - results/latest.json is absent (bench not run on this machine), // - latest.json is older than renderPerfMaxAge (the bench predates recent // work and proves nothing about it), // - latest.json predates the committed baseline (stale result from before // the current baseline was captured). // In every SKIP case it says how to produce a fresh result (run the // `render-bench` suite from dojo). import ( "encoding/json" "fmt" "os" "time" ) const ( // renderPerfP50Threshold fails the gate when home-page p50 exceeds the // baseline p50 by more than this fraction. 25% sits above observed // run-to-run variance (~7%) but well below any real regression worth a // human look (WO-RP-013 mandated default). renderPerfP50Threshold = 0.25 // renderPerfP95Threshold is the secondary tail gate. It WARNs rather than // FAILs: observed p95 on an otherwise-idle dev instance varies >100% // between runs minutes apart (36ms → 76ms) while p50 stays within ~7%, // so a hard p95 gate would block commits on ambient dev-stack noise. // p50 is the blocking metric. renderPerfP95Threshold = 0.40 // renderPerfMaxAge is how old a latest.json may be and still count as // evidence about the current tree. renderPerfMaxAge = 7 * 24 * time.Hour ) type renderBenchPage struct { Page string `json:"page"` Path string `json:"path"` Gating bool `json:"gating"` P50Ms float64 `json:"p50_ms"` P95Ms float64 `json:"p95_ms"` } type renderBenchResults struct { Timestamp string `json:"timestamp"` Instance string `json:"instance"` Pages []renderBenchPage `json:"pages"` } func loadRenderBench(path string) (*renderBenchResults, error) { raw, err := os.ReadFile(path) if err != nil { return nil, err } var r renderBenchResults if err := json.Unmarshal(raw, &r); err != nil { return nil, fmt.Errorf("%s: %w", path, err) } return &r, nil } func (r *renderBenchResults) gatingPage() *renderBenchPage { for i := range r.Pages { if r.Pages[i].Gating { return &r.Pages[i] } } return nil } func (r *renderBenchResults) parsedTime() (time.Time, error) { return time.Parse(time.RFC3339, r.Timestamp) } // renderPerfVerdict is the outcome of one baseline/latest comparison, // separated from the Reporter so it can be unit-tested. type renderPerfVerdict struct { skip bool warn bool fail bool message string } func compareRenderPerf(baseline, latest *renderBenchResults, now time.Time) renderPerfVerdict { base := baseline.gatingPage() if base == nil { return renderPerfVerdict{skip: true, message: "baseline.json has no gating page — regenerate it from the render-bench suite"} } cur := latest.gatingPage() if cur == nil { return renderPerfVerdict{skip: true, message: "latest.json has no gating page — re-run the render-bench suite from dojo"} } latestAt, err := latest.parsedTime() if err != nil { return renderPerfVerdict{skip: true, message: fmt.Sprintf("latest.json has an unparseable timestamp (%v) — re-run the render-bench suite from dojo", err)} } if now.Sub(latestAt) > renderPerfMaxAge { return renderPerfVerdict{skip: true, message: fmt.Sprintf("latest bench result is %.0fh old (max %.0fh) — run the render-bench suite from dojo for a fresh result", now.Sub(latestAt).Hours(), renderPerfMaxAge.Hours())} } if baseAt, err := baseline.parsedTime(); err == nil && latestAt.Before(baseAt) { return renderPerfVerdict{skip: true, message: "latest bench result predates the committed baseline — run the render-bench suite from dojo"} } if base.P50Ms <= 0 { return renderPerfVerdict{skip: true, message: "baseline gating p50 is zero — regenerate baseline.json from the render-bench suite"} } p50Delta := (cur.P50Ms - base.P50Ms) / base.P50Ms p95Delta := 0.0 if base.P95Ms > 0 { p95Delta = (cur.P95Ms - base.P95Ms) / base.P95Ms } if p50Delta > renderPerfP50Threshold { return renderPerfVerdict{fail: true, message: fmt.Sprintf( "Home page render is %.0f%% slower than baseline (p50 %.1fms vs %.1fms). Investigate before proceeding — profile the regression (see docs/works/WO-RP-001 methodology); do NOT raise the baseline to make this pass without Captain approval.", p50Delta*100, cur.P50Ms, base.P50Ms)} } if p95Delta > renderPerfP95Threshold { return renderPerfVerdict{warn: true, message: fmt.Sprintf( "Home page render tail is %.0f%% slower than baseline (p95 %.1fms vs %.1fms; p50 healthy at %.1fms vs %.1fms). p95 is noisy on the shared dev stack — re-run the render-bench suite; investigate if it persists.", p95Delta*100, cur.P95Ms, base.P95Ms, cur.P50Ms, base.P50Ms)} } return renderPerfVerdict{message: fmt.Sprintf("home p50 %.1fms vs baseline %.1fms (%+.0f%%), p95 %.1fms vs %.1fms (%+.0f%%)", cur.P50Ms, base.P50Ms, p50Delta*100, cur.P95Ms, base.P95Ms, p95Delta*100)} }