check-safety/golden_test.go

package main

// Golden characterization test for cmd/check-safety.
//
// Builds the tool once (TestMain), then runs it against small in-memory fixture
// trees created per test case.  Normalises volatile tokens (absolute fixture
// path, SDK version) before comparing against committed golden files.
//
// Usage:
//
//	go test ./cmd/check-safety/ -run TestGoldenCharacterization          # compare
//	go test ./cmd/check-safety/ -run TestGoldenCharacterization -update  # regenerate
//
// Archetype-B checks (2f codegen, 3 Go lint, 3b orchestrator tests, 4 frontend
// lint) are intentionally excluded from the golden corpus: they shell out to
// external tools and cannot be snapshotted deterministically.  Fixtures are
// arranged so those four checks print SKIP (or OK for the `clean` case where
// the minimal Go package is deterministically lint-clean).

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"testing"
)

var (
	update          = flag.Bool("update", false, "regenerate golden files instead of comparing")
	checkSafetyBin  string // set by TestMain
)

// fixtureDef describes a single fixture tree that is created in a temp dir.
type fixtureDef struct {
	// path is relative to the fixture root; content is the file content.
	path    string
	content string
}

// goldenCase is one row in the characterisation matrix.
type goldenCase struct {
	name     string
	fixtures []fixtureDef
	// extraArgs are additional CLI args passed after the positional target dir.
	extraArgs []string
	wantExit  int
}

// goldenCases enumerates the test matrix.
var goldenCases = []goldenCase{
	{
		// nomod: no go.mod present.
		//   — Check 1  FAIL: os.Getenv(secret) outside config
		//   — Check 2  SKIP: no proto procedures
		//   — Check 2b OK
		//   — Check 2c FAIL: missing go.mod
		//   — Check 3  SKIP: no go.mod found (external tools NOT invoked)
		//   — Check 3b SKIP: orchestrator not found
		//   — Check 2f SKIP: no codegen targets
		//   — Check 4  SKIP: no frontend sources
		//   — Check 14 FAIL: raw SQL
		//   — Check 15 FAIL: err.Error() leaked
		//   — Check 17 FAIL: TODO marker
		name: "nomod",
		fixtures: []fixtureDef{
			{
				path: "internal/service/handler.go",
				content: `// Package service provides a service handler.
package service

import (
	"database/sql"
	"net/http"
	"os"
)

// MyService handles requests.
type MyService struct{}

// Init initialises the service.
// TODO: add proper initialisation
func (s *MyService) Init() {
	// Leak a secret env var outside config.Load() — triggers Check 1
	_ = os.Getenv("JWT_SECRET")
}

// QueryUsers fetches users using raw SQL — triggers Check 14.
func (s *MyService) QueryUsers(db *sql.DB) {
	db.QueryContext(nil, "SELECT id, name FROM users WHERE active = true") //nolint
}

// HandleError leaks err.Error() to HTTP client — triggers Check 15.
func (s *MyService) HandleError(w http.ResponseWriter, err error) {
	http.Error(w, err.Error(), http.StatusInternalServerError)
}
`,
			},
		},
		wantExit: 1,
	},
	{
		// clean: go.mod present, minimal valid Go package, no violations.
		//   — All deterministic checks OK or SKIP
		//   — Check 2c OK: go.mod has no replace directive for block/core
		//   — Check 3  OK: lint pipeline passes (deterministic: trivial package)
		//   — Check 3b SKIP
		//   — Check 2f SKIP
		//   — Check 4  SKIP: no frontend sources
		name: "clean",
		fixtures: []fixtureDef{
			{
				path:    "go.mod",
				content: "module example.com/goldenclean\n\ngo 1.21\n",
			},
			{
				path: "internal/pkg/util.go",
				content: `// Package pkg provides utility functions.
package pkg

// Add returns the sum of a and b.
func Add(a, b int) int {
	return a + b
}
`,
			},
		},
		wantExit: 0,
	},
}

// ---- TestMain: build the binary once ----------------------------------------

func TestMain(m *testing.M) {
	flag.Parse()

	// Build check-safety to a temp file.
	tmpDir, err := os.MkdirTemp("", "check-safety-golden-bin-*")
	if err != nil {
		fmt.Fprintf(os.Stderr, "golden_test: failed to create tmpdir for binary: %v\n", err)
		os.Exit(2)
	}
	defer func() { _ = os.RemoveAll(tmpDir) }()

	binPath := filepath.Join(tmpDir, "check-safety")
	// Build the standalone check-safety binary from its own module.
	buildCmd := exec.Command("go", "build", "-o", binPath, ".")
	buildCmd.Env = os.Environ() // inherit env (GOPATH, GOMODCACHE, etc.)
	if out, buildErr := buildCmd.CombinedOutput(); buildErr != nil {
		fmt.Fprintf(os.Stderr, "golden_test: failed to build check-safety binary:\n%s\n", out)
		os.Exit(2)
	}
	checkSafetyBin = binPath

	os.Exit(m.Run())
}

// ---- TestGoldenCharacterization -----------------------------------------------

func TestGoldenCharacterization(t *testing.T) {
	for _, tc := range goldenCases {
		t.Run(tc.name, func(t *testing.T) {
			// 1. Create fixture tree in a temp dir.
			fixtureDir := t.TempDir()
			for _, f := range tc.fixtures {
				dst := filepath.Join(fixtureDir, filepath.FromSlash(f.path))
				if mkErr := os.MkdirAll(filepath.Dir(dst), 0o755); mkErr != nil {
					t.Fatalf("mkdir %s: %v", filepath.Dir(dst), mkErr)
				}
				if writeErr := os.WriteFile(dst, []byte(f.content), 0o644); writeErr != nil {
					t.Fatalf("write fixture %s: %v", dst, writeErr)
				}
			}

			// 2. Run check-safety against the fixture.
			args := append([]string{fixtureDir}, tc.extraArgs...)
			cmd := exec.Command(checkSafetyBin, args...)
			raw, _ := cmd.CombinedOutput() // exit code is checked separately
			gotExit := 0
			if cmd.ProcessState != nil {
				gotExit = cmd.ProcessState.ExitCode()
			}

			// 3. Normalise volatile tokens.
			got := normalizeGoldenOutput(string(raw), fixtureDir)

			// 4. Golden update mode: write files and return.
			if *update {
				writeGolden(t, tc.name, got, gotExit)
				return
			}

			// 5. Compare exit code.
			if gotExit != tc.wantExit {
				t.Errorf("exit code = %d, want %d", gotExit, tc.wantExit)
			}

			// 6. Compare normalised stdout.
			want := readGoldenStdout(t, tc.name)
			if got != want {
				t.Errorf("golden mismatch for case %q\n\n--- want ---\n%s\n--- got ---\n%s\n",
					tc.name, want, got)
			}

			// 7. Compare exit code from golden file.
			wantExitFromFile := readGoldenExit(t, tc.name)
			if gotExit != wantExitFromFile {
				t.Errorf("exit code = %d, want %d (from golden file)", gotExit, wantExitFromFile)
			}
		})
	}
}

// ---- normalisation ----------------------------------------------------------

// reSDKVersion matches the block/core semver in Check-2c messages.
// Example: "require git.dev.alexdunmow.com/block/core v0.9.0,"
var reSDKVersion = regexp.MustCompile(
	`(git\.dev\.alexdunmow\.com/block/core) v\d+\.\d+\.\d+[-\w]*`,
)

// normalizeGoldenOutput replaces volatile tokens with stable placeholders.
//   - The absolute fixture directory path → FIXTURE_DIR
//   - The SDK semver in Check 2c messages → <SDK_VERSION>
func normalizeGoldenOutput(output, fixtureDir string) string {
	// Normalise all occurrences of the absolute fixture path.
	out := strings.ReplaceAll(output, fixtureDir, "FIXTURE_DIR")
	// Normalise SDK version so golden files survive SDK bumps.
	out = reSDKVersion.ReplaceAllString(out, "$1 <SDK_VERSION>")
	return out
}

// ---- golden file helpers ----------------------------------------------------

func goldenDir(name string) string {
	return filepath.Join("testdata", "golden", name)
}

func goldenStdoutPath(name string) string {
	return filepath.Join(goldenDir(name), "expected.stdout")
}

func goldenExitPath(name string) string {
	return filepath.Join(goldenDir(name), "expected.exit")
}

func readGoldenStdout(t *testing.T, name string) string {
	t.Helper()
	data, err := os.ReadFile(goldenStdoutPath(name))
	if err != nil {
		t.Fatalf("read golden stdout for %q: %v (run with -update to generate)", name, err)
	}
	return string(data)
}

func readGoldenExit(t *testing.T, name string) int {
	t.Helper()
	data, err := os.ReadFile(goldenExitPath(name))
	if err != nil {
		t.Fatalf("read golden exit for %q: %v (run with -update to generate)", name, err)
	}
	trimmed := strings.TrimSpace(string(data))
	if trimmed == "0" {
		return 0
	}
	if trimmed == "1" {
		return 1
	}
	t.Fatalf("unexpected exit code in golden file for %q: %q", name, trimmed)
	return -1
}

func writeGolden(t *testing.T, name, stdout string, exitCode int) {
	t.Helper()
	dir := goldenDir(name)
	if err := os.MkdirAll(dir, 0o755); err != nil {
		t.Fatalf("mkdir golden dir %s: %v", dir, err)
	}
	if err := os.WriteFile(goldenStdoutPath(name), []byte(stdout), 0o644); err != nil {
		t.Fatalf("write golden stdout for %q: %v", name, err)
	}
	exitStr := fmt.Sprintf("%d\n", exitCode)
	if err := os.WriteFile(goldenExitPath(name), []byte(exitStr), 0o644); err != nil {
		t.Fatalf("write golden exit for %q: %v", name, err)
	}
	t.Logf("golden files updated for case %q (exit=%d)", name, exitCode)
}