package main // Golden characterization test for cmd/check-safety. // // Builds the tool once (TestMain), then runs it against small in-memory fixture // trees created per test case. Normalises volatile tokens (absolute fixture // path, SDK version) before comparing against committed golden files. // // Usage: // // go test ./cmd/check-safety/ -run TestGoldenCharacterization # compare // go test ./cmd/check-safety/ -run TestGoldenCharacterization -update # regenerate // // Archetype-B checks (2f codegen, 3 Go lint, 3b orchestrator tests, 4 frontend // lint) are intentionally excluded from the golden corpus: they shell out to // external tools and cannot be snapshotted deterministically. Fixtures are // arranged so those four checks print SKIP (or OK for the `clean` case where // the minimal Go package is deterministically lint-clean). import ( "flag" "fmt" "os" "os/exec" "path/filepath" "regexp" "strings" "testing" ) var ( update = flag.Bool("update", false, "regenerate golden files instead of comparing") checkSafetyBin string // set by TestMain ) // fixtureDef describes a single fixture tree that is created in a temp dir. type fixtureDef struct { // path is relative to the fixture root; content is the file content. path string content string } // goldenCase is one row in the characterisation matrix. type goldenCase struct { name string fixtures []fixtureDef // extraArgs are additional CLI args passed after the positional target dir. extraArgs []string wantExit int } // goldenCases enumerates the test matrix. var goldenCases = []goldenCase{ { // nomod: no go.mod present. // — Check 1 FAIL: os.Getenv(secret) outside config // — Check 2 SKIP: no proto procedures // — Check 2b OK // — Check 2c FAIL: missing go.mod // — Check 3 SKIP: no go.mod found (external tools NOT invoked) // — Check 3b SKIP: orchestrator not found // — Check 2f SKIP: no codegen targets // — Check 4 SKIP: no frontend sources // — Check 14 FAIL: raw SQL // — Check 15 FAIL: err.Error() leaked // — Check 17 FAIL: TODO marker name: "nomod", fixtures: []fixtureDef{ { path: "internal/service/handler.go", content: `// Package service provides a service handler. package service import ( "database/sql" "net/http" "os" ) // MyService handles requests. type MyService struct{} // Init initialises the service. // TODO: add proper initialisation func (s *MyService) Init() { // Leak a secret env var outside config.Load() — triggers Check 1 _ = os.Getenv("JWT_SECRET") } // QueryUsers fetches users using raw SQL — triggers Check 14. func (s *MyService) QueryUsers(db *sql.DB) { db.QueryContext(nil, "SELECT id, name FROM users WHERE active = true") //nolint } // HandleError leaks err.Error() to HTTP client — triggers Check 15. func (s *MyService) HandleError(w http.ResponseWriter, err error) { http.Error(w, err.Error(), http.StatusInternalServerError) } `, }, }, wantExit: 1, }, { // clean: go.mod present, minimal valid Go package, no violations. // — All deterministic checks OK or SKIP // — Check 2c OK: go.mod has no replace directive for block/core // — Check 3 OK: lint pipeline passes (deterministic: trivial package) // — Check 3b SKIP // — Check 2f SKIP // — Check 4 SKIP: no frontend sources name: "clean", fixtures: []fixtureDef{ { path: "go.mod", content: "module example.com/goldenclean\n\ngo 1.21\n", }, { path: "internal/pkg/util.go", content: `// Package pkg provides utility functions. package pkg // Add returns the sum of a and b. func Add(a, b int) int { return a + b } `, }, }, wantExit: 0, }, } // ---- TestMain: build the binary once ---------------------------------------- func TestMain(m *testing.M) { flag.Parse() // Build check-safety to a temp file. tmpDir, err := os.MkdirTemp("", "check-safety-golden-bin-*") if err != nil { fmt.Fprintf(os.Stderr, "golden_test: failed to create tmpdir for binary: %v\n", err) os.Exit(2) } defer func() { _ = os.RemoveAll(tmpDir) }() binPath := filepath.Join(tmpDir, "check-safety") // Build the standalone check-safety binary from its own module. buildCmd := exec.Command("go", "build", "-o", binPath, ".") buildCmd.Env = os.Environ() // inherit env (GOPATH, GOMODCACHE, etc.) if out, buildErr := buildCmd.CombinedOutput(); buildErr != nil { fmt.Fprintf(os.Stderr, "golden_test: failed to build check-safety binary:\n%s\n", out) os.Exit(2) } checkSafetyBin = binPath os.Exit(m.Run()) } // ---- TestGoldenCharacterization ----------------------------------------------- func TestGoldenCharacterization(t *testing.T) { for _, tc := range goldenCases { t.Run(tc.name, func(t *testing.T) { // 1. Create fixture tree in a temp dir. fixtureDir := t.TempDir() for _, f := range tc.fixtures { dst := filepath.Join(fixtureDir, filepath.FromSlash(f.path)) if mkErr := os.MkdirAll(filepath.Dir(dst), 0o755); mkErr != nil { t.Fatalf("mkdir %s: %v", filepath.Dir(dst), mkErr) } if writeErr := os.WriteFile(dst, []byte(f.content), 0o644); writeErr != nil { t.Fatalf("write fixture %s: %v", dst, writeErr) } } // 2. Run check-safety against the fixture. args := append([]string{fixtureDir}, tc.extraArgs...) cmd := exec.Command(checkSafetyBin, args...) raw, _ := cmd.CombinedOutput() // exit code is checked separately gotExit := 0 if cmd.ProcessState != nil { gotExit = cmd.ProcessState.ExitCode() } // 3. Normalise volatile tokens. got := normalizeGoldenOutput(string(raw), fixtureDir) // 4. Golden update mode: write files and return. if *update { writeGolden(t, tc.name, got, gotExit) return } // 5. Compare exit code. if gotExit != tc.wantExit { t.Errorf("exit code = %d, want %d", gotExit, tc.wantExit) } // 6. Compare normalised stdout. want := readGoldenStdout(t, tc.name) if got != want { t.Errorf("golden mismatch for case %q\n\n--- want ---\n%s\n--- got ---\n%s\n", tc.name, want, got) } // 7. Compare exit code from golden file. wantExitFromFile := readGoldenExit(t, tc.name) if gotExit != wantExitFromFile { t.Errorf("exit code = %d, want %d (from golden file)", gotExit, wantExitFromFile) } }) } } // ---- normalisation ---------------------------------------------------------- // reSDKVersion matches the block/core semver in Check-2c messages. // Example: "require git.dev.alexdunmow.com/block/core v0.9.0," var reSDKVersion = regexp.MustCompile( `(git\.dev\.alexdunmow\.com/block/core) v\d+\.\d+\.\d+[-\w]*`, ) // normalizeGoldenOutput replaces volatile tokens with stable placeholders. // - The absolute fixture directory path → FIXTURE_DIR // - The SDK semver in Check 2c messages → func normalizeGoldenOutput(output, fixtureDir string) string { // Normalise all occurrences of the absolute fixture path. out := strings.ReplaceAll(output, fixtureDir, "FIXTURE_DIR") // Normalise SDK version so golden files survive SDK bumps. out = reSDKVersion.ReplaceAllString(out, "$1 ") return out } // ---- golden file helpers ---------------------------------------------------- func goldenDir(name string) string { return filepath.Join("testdata", "golden", name) } func goldenStdoutPath(name string) string { return filepath.Join(goldenDir(name), "expected.stdout") } func goldenExitPath(name string) string { return filepath.Join(goldenDir(name), "expected.exit") } func readGoldenStdout(t *testing.T, name string) string { t.Helper() data, err := os.ReadFile(goldenStdoutPath(name)) if err != nil { t.Fatalf("read golden stdout for %q: %v (run with -update to generate)", name, err) } return string(data) } func readGoldenExit(t *testing.T, name string) int { t.Helper() data, err := os.ReadFile(goldenExitPath(name)) if err != nil { t.Fatalf("read golden exit for %q: %v (run with -update to generate)", name, err) } trimmed := strings.TrimSpace(string(data)) if trimmed == "0" { return 0 } if trimmed == "1" { return 1 } t.Fatalf("unexpected exit code in golden file for %q: %q", name, trimmed) return -1 } func writeGolden(t *testing.T, name, stdout string, exitCode int) { t.Helper() dir := goldenDir(name) if err := os.MkdirAll(dir, 0o755); err != nil { t.Fatalf("mkdir golden dir %s: %v", dir, err) } if err := os.WriteFile(goldenStdoutPath(name), []byte(stdout), 0o644); err != nil { t.Fatalf("write golden stdout for %q: %v", name, err) } exitStr := fmt.Sprintf("%d\n", exitCode) if err := os.WriteFile(goldenExitPath(name), []byte(exitStr), 0o644); err != nil { t.Fatalf("write golden exit for %q: %v", name, err) } t.Logf("golden files updated for case %q (exit=%d)", name, exitCode) }