check-safety/renderperf.go

package main

// Render-performance regression gate (WO-RP-013).
//
// Compares the latest e2e render benchmark result (cms
// e2e/render-bench/results/latest.json, written by the dojo suite
// `render-bench`) against the committed baseline
// (e2e/render-bench/baseline.json) and FAILS when the home page — the gating
// page class — is markedly slower.
//
// Staleness rule: this check must never block unrelated work when no live
// stack is running. It SKIPs (never fails) when:
//   - baseline.json is absent (repo is not the cms repo, or gate not set up),
//   - results/latest.json is absent (bench not run on this machine),
//   - latest.json is older than renderPerfMaxAge (the bench predates recent
//     work and proves nothing about it),
//   - latest.json predates the committed baseline (stale result from before
//     the current baseline was captured).
// In every SKIP case it says how to produce a fresh result (run the
// `render-bench` suite from dojo).

import (
	"encoding/json"
	"fmt"
	"os"
	"time"
)

const (
	// renderPerfP50Threshold fails the gate when home-page p50 exceeds the
	// baseline p50 by more than this fraction. 25% sits above observed
	// run-to-run variance (~7%) but well below any real regression worth a
	// human look (WO-RP-013 mandated default).
	renderPerfP50Threshold = 0.25
	// renderPerfP95Threshold is the secondary tail gate. It WARNs rather than
	// FAILs: observed p95 on an otherwise-idle dev instance varies >100%
	// between runs minutes apart (36ms → 76ms) while p50 stays within ~7%,
	// so a hard p95 gate would block commits on ambient dev-stack noise.
	// p50 is the blocking metric.
	renderPerfP95Threshold = 0.40
	// renderPerfMaxAge is how old a latest.json may be and still count as
	// evidence about the current tree.
	renderPerfMaxAge = 7 * 24 * time.Hour
)

type renderBenchPage struct {
	Page   string  `json:"page"`
	Path   string  `json:"path"`
	Gating bool    `json:"gating"`
	P50Ms  float64 `json:"p50_ms"`
	P95Ms  float64 `json:"p95_ms"`
}

type renderBenchResults struct {
	Timestamp string            `json:"timestamp"`
	Instance  string            `json:"instance"`
	Pages     []renderBenchPage `json:"pages"`
}

func loadRenderBench(path string) (*renderBenchResults, error) {
	raw, err := os.ReadFile(path)
	if err != nil {
		return nil, err
	}
	var r renderBenchResults
	if err := json.Unmarshal(raw, &r); err != nil {
		return nil, fmt.Errorf("%s: %w", path, err)
	}
	return &r, nil
}

func (r *renderBenchResults) gatingPage() *renderBenchPage {
	for i := range r.Pages {
		if r.Pages[i].Gating {
			return &r.Pages[i]
		}
	}
	return nil
}

func (r *renderBenchResults) parsedTime() (time.Time, error) {
	return time.Parse(time.RFC3339, r.Timestamp)
}

// renderPerfVerdict is the outcome of one baseline/latest comparison,
// separated from the Reporter so it can be unit-tested.
type renderPerfVerdict struct {
	skip    bool
	warn    bool
	fail    bool
	message string
}

func compareRenderPerf(baseline, latest *renderBenchResults, now time.Time) renderPerfVerdict {
	base := baseline.gatingPage()
	if base == nil {
		return renderPerfVerdict{skip: true, message: "baseline.json has no gating page — regenerate it from the render-bench suite"}
	}
	cur := latest.gatingPage()
	if cur == nil {
		return renderPerfVerdict{skip: true, message: "latest.json has no gating page — re-run the render-bench suite from dojo"}
	}

	latestAt, err := latest.parsedTime()
	if err != nil {
		return renderPerfVerdict{skip: true, message: fmt.Sprintf("latest.json has an unparseable timestamp (%v) — re-run the render-bench suite from dojo", err)}
	}
	if now.Sub(latestAt) > renderPerfMaxAge {
		return renderPerfVerdict{skip: true, message: fmt.Sprintf("latest bench result is %.0fh old (max %.0fh) — run the render-bench suite from dojo for a fresh result", now.Sub(latestAt).Hours(), renderPerfMaxAge.Hours())}
	}
	if baseAt, err := baseline.parsedTime(); err == nil && latestAt.Before(baseAt) {
		return renderPerfVerdict{skip: true, message: "latest bench result predates the committed baseline — run the render-bench suite from dojo"}
	}

	if base.P50Ms <= 0 {
		return renderPerfVerdict{skip: true, message: "baseline gating p50 is zero — regenerate baseline.json from the render-bench suite"}
	}

	p50Delta := (cur.P50Ms - base.P50Ms) / base.P50Ms
	p95Delta := 0.0
	if base.P95Ms > 0 {
		p95Delta = (cur.P95Ms - base.P95Ms) / base.P95Ms
	}

	if p50Delta > renderPerfP50Threshold {
		return renderPerfVerdict{fail: true, message: fmt.Sprintf(
			"Home page render is %.0f%% slower than baseline (p50 %.1fms vs %.1fms). Investigate before proceeding — profile the regression (see docs/works/WO-RP-001 methodology); do NOT raise the baseline to make this pass without Captain approval.",
			p50Delta*100, cur.P50Ms, base.P50Ms)}
	}
	if p95Delta > renderPerfP95Threshold {
		return renderPerfVerdict{warn: true, message: fmt.Sprintf(
			"Home page render tail is %.0f%% slower than baseline (p95 %.1fms vs %.1fms; p50 healthy at %.1fms vs %.1fms). p95 is noisy on the shared dev stack — re-run the render-bench suite; investigate if it persists.",
			p95Delta*100, cur.P95Ms, base.P95Ms, cur.P50Ms, base.P50Ms)}
	}

	return renderPerfVerdict{message: fmt.Sprintf("home p50 %.1fms vs baseline %.1fms (%+.0f%%), p95 %.1fms vs %.1fms (%+.0f%%)",
		cur.P50Ms, base.P50Ms, p50Delta*100, cur.P95Ms, base.P95Ms, p95Delta*100)}
}