8d844f2d61
Compares cms e2e/render-bench/results/latest.json (written by the dojo render-bench suite) against the committed baseline.json. FAILs when home-page p50 exceeds baseline by >25%, with an explicit instruction to investigate and never bump the baseline without Captain approval. p95 breach (>40%) WARNs only: measured ambient p95 variance on an idle dev instance exceeds 100% between runs while p50 holds within ~7%, so a hard tail gate would block unrelated commits on dev-stack noise. SKIPs (never blocks) when: baseline.json absent (non-cms repo), latest.json absent (bench not run), latest older than 7 days, or latest predates the baseline — each with a nudge to run the suite from dojo. Comparison logic is Reporter-free and unit-tested (renderperf_test.go). Verified against the live tree in all three states: OK (p50 22.9 vs 27.3ms), FAIL (26% slower via delay proxy, mandated message), SKIP (latest.json removed). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
77 lines
2.4 KiB
Go
77 lines
2.4 KiB
Go
package main
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func benchFixture(ts string, p50, p95 float64) *renderBenchResults {
|
|
return &renderBenchResults{
|
|
Timestamp: ts,
|
|
Instance: "blockninjacms.blockninja.dev",
|
|
Pages: []renderBenchPage{
|
|
{Page: "home", Path: "/", Gating: true, P50Ms: p50, P95Ms: p95},
|
|
{Page: "blog-post", Path: "/blog/x", P50Ms: 18, P95Ms: 25},
|
|
},
|
|
}
|
|
}
|
|
|
|
func TestCompareRenderPerf(t *testing.T) {
|
|
now := time.Date(2026, 7, 7, 12, 0, 0, 0, time.UTC)
|
|
baseline := benchFixture("2026-07-06T16:47:00Z", 27.3, 37.0)
|
|
|
|
t.Run("within threshold passes", func(t *testing.T) {
|
|
v := compareRenderPerf(baseline, benchFixture("2026-07-07T10:00:00Z", 30.0, 40.0), now)
|
|
if v.skip || v.fail {
|
|
t.Fatalf("expected pass, got %+v", v)
|
|
}
|
|
})
|
|
|
|
t.Run("p50 regression fails with investigate message", func(t *testing.T) {
|
|
v := compareRenderPerf(baseline, benchFixture("2026-07-07T10:00:00Z", 40.0, 41.0), now)
|
|
if !v.fail {
|
|
t.Fatalf("expected fail, got %+v", v)
|
|
}
|
|
for _, want := range []string{"slower than baseline", "Investigate before proceeding", "do NOT raise the baseline", "Captain approval"} {
|
|
if !strings.Contains(v.message, want) {
|
|
t.Errorf("fail message missing %q: %s", want, v.message)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("p95-only regression warns via secondary gate", func(t *testing.T) {
|
|
v := compareRenderPerf(baseline, benchFixture("2026-07-07T10:00:00Z", 28.0, 60.0), now)
|
|
if !v.warn || v.fail {
|
|
t.Fatalf("expected warn (not fail) for p95-only breach, got %+v", v)
|
|
}
|
|
if !strings.Contains(v.message, "p95") {
|
|
t.Errorf("expected p95 metric in message: %s", v.message)
|
|
}
|
|
})
|
|
|
|
t.Run("stale latest skips", func(t *testing.T) {
|
|
v := compareRenderPerf(baseline, benchFixture("2026-06-20T10:00:00Z", 40.0, 60.0), now)
|
|
if !v.skip {
|
|
t.Fatalf("expected skip for stale result, got %+v", v)
|
|
}
|
|
if !strings.Contains(v.message, "dojo") {
|
|
t.Errorf("skip message should say how to refresh: %s", v.message)
|
|
}
|
|
})
|
|
|
|
t.Run("latest predating baseline skips", func(t *testing.T) {
|
|
v := compareRenderPerf(baseline, benchFixture("2026-07-06T10:00:00Z", 40.0, 60.0), now)
|
|
if !v.skip {
|
|
t.Fatalf("expected skip for pre-baseline result, got %+v", v)
|
|
}
|
|
})
|
|
|
|
t.Run("boundary: exactly 25 percent passes", func(t *testing.T) {
|
|
v := compareRenderPerf(baseline, benchFixture("2026-07-07T10:00:00Z", 27.3*1.25, 37.0), now)
|
|
if v.fail {
|
|
t.Fatalf("exactly-threshold should not fail: %+v", v)
|
|
}
|
|
})
|
|
}
|