From ad6d87bf06bd853324698d836e539af2b7a62d09 Mon Sep 17 00:00:00 2001 From: Alex Dunmow Date: Fri, 3 Jul 2026 18:34:17 +0800 Subject: [PATCH] =?UTF-8?q?feat(ninja):=20`plugin=20build`/`verify`=20?= =?UTF-8?q?=E2=86=92=20.bnp=20packer=20+=20ABI=20riders=20(WO-WZ-009)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ninja plugin build` compiles a plugin to reactor-mode wasip1 wasm (Go >= 1.24 enforced), extracts manifest.pb by driving one HOOK_DESCRIBE over wazero with failing host stubs, and packs a tar.zst .bnp (plugin.wasm, plugin.mod, manifest.pb + migrations/schemas/assets/web-dist when present) with a summary table. A describe-time capability call (e.g. db.* from Register) fails with an actionable error naming the offending method. `ninja plugin verify` re-runs the CMS reader's layout/name/abi/path-safety/size checks standalone (deliberate duplication of cms backend/plugin/bnp/reader.go; kept in lockstep by WO-WZ-010). ABI riders (additive; buf breaking clean): - ABI_ERROR_CODE_TX_EXPIRED enum value + bnwasm guest mapping to a new bnwasm.ErrTxExpired sentinel (retryable tx expiry, distinct from real faults); the cms dbexec side adopts the emit separately. - PluginManifest.data_dir bool + a first-class `data_dir` key on the plugin.mod parser (so writeMod's struct round-trip can't drop it); `plugin build` stamps it from plugin.mod into the manifest. Docs: wasm-abi.md gains a Building & packing section, the error-code table row, the manifest data_dir mapping, and the plugin.mod reference. Tests: CLI e2e builds the WZ-002 fixture → verify + manifest block keys; a capfixture proves the actionable describe-time error; verify rejects each malformed class; bnwasm TX_EXPIRED classification. Co-Authored-By: Claude Fable 5 --- abi/proto/v1/invoke.proto | 6 + abi/proto/v1/manifest.proto | 7 + abi/v1/invoke.pb.go | 13 +- abi/v1/manifest.pb.go | 22 +- cmd/ninja/cmd/plugin.go | 5 + cmd/ninja/cmd/plugin_build.go | 119 ++++++++ cmd/ninja/cmd/plugin_build_test.go | 279 ++++++++++++++++++ cmd/ninja/internal/bnp/build.go | 248 ++++++++++++++++ cmd/ninja/internal/bnp/manifest.go | 171 +++++++++++ cmd/ninja/internal/bnp/pack.go | 138 +++++++++ cmd/ninja/internal/bnp/verify.go | 275 +++++++++++++++++ docs/wasm-abi.md | 66 +++++ plugin/mod.go | 8 + plugin/mod_test.go | 31 ++ plugin/wasmguest/bnwasm/transport.go | 40 ++- plugin/wasmguest/bnwasm/tx_expired_test.go | 74 +++++ plugin/wasmguest/testdata/capfixture/main.go | 8 + .../wasmguest/testdata/capfixture/plugin.mod | 6 + .../testdata/capfixture/registration.go | 38 +++ plugin/wasmguest/testdata/fixture/plugin.mod | 8 + 20 files changed, 1551 insertions(+), 11 deletions(-) create mode 100644 cmd/ninja/cmd/plugin_build.go create mode 100644 cmd/ninja/cmd/plugin_build_test.go create mode 100644 cmd/ninja/internal/bnp/build.go create mode 100644 cmd/ninja/internal/bnp/manifest.go create mode 100644 cmd/ninja/internal/bnp/pack.go create mode 100644 cmd/ninja/internal/bnp/verify.go create mode 100644 plugin/wasmguest/bnwasm/tx_expired_test.go create mode 100644 plugin/wasmguest/testdata/capfixture/main.go create mode 100644 plugin/wasmguest/testdata/capfixture/plugin.mod create mode 100644 plugin/wasmguest/testdata/capfixture/registration.go create mode 100644 plugin/wasmguest/testdata/fixture/plugin.mod diff --git a/abi/proto/v1/invoke.proto b/abi/proto/v1/invoke.proto index 7accfbc..69109b6 100644 --- a/abi/proto/v1/invoke.proto +++ b/abi/proto/v1/invoke.proto @@ -76,6 +76,12 @@ enum AbiErrorCode { ABI_ERROR_CODE_DEADLINE_EXCEEDED = 4; // The caller is not entitled to this capability. ABI_ERROR_CODE_PERMISSION_DENIED = 5; + // A db.* call named a transaction handle the host has already expired + // (dropped at the call-chain deadline, WO-WZ-007). Distinct from a real + // fault: the unit of work is retryable in a fresh transaction. Emitted by + // the cms dbexec side (adopted separately) and mapped guest-side to + // bnwasm.ErrTxExpired. + ABI_ERROR_CODE_TX_EXPIRED = 6; } // AbiError is the structured error carried by InvokeResponse and diff --git a/abi/proto/v1/manifest.proto b/abi/proto/v1/manifest.proto index 0fe33df..8e75993 100644 --- a/abi/proto/v1/manifest.proto +++ b/abi/proto/v1/manifest.proto @@ -97,6 +97,13 @@ message PluginManifest { // (CoreServiceBindings.Bind becomes a static declaration; the host // constructs and mounts the handlers). repeated CoreServiceBinding core_service_bindings = 29; + + // data_dir requests a persistent per-plugin /data preopen at load. Its + // source of truth is the plugin.mod `data_dir = true` key, which lives + // outside the guest code, so DESCRIBE cannot populate it: the packer + // (`ninja plugin build`) stamps it into the manifest from plugin.mod so + // the loader reads one source. OFF by default. + bool data_dir = 30; } // Dependency mirrors plugin.Dependency. diff --git a/abi/v1/invoke.pb.go b/abi/v1/invoke.pb.go index 2b2297f..dfcc2a1 100644 --- a/abi/v1/invoke.pb.go +++ b/abi/v1/invoke.pb.go @@ -131,6 +131,12 @@ const ( AbiErrorCode_ABI_ERROR_CODE_DEADLINE_EXCEEDED AbiErrorCode = 4 // The caller is not entitled to this capability. AbiErrorCode_ABI_ERROR_CODE_PERMISSION_DENIED AbiErrorCode = 5 + // A db.* call named a transaction handle the host has already expired + // (dropped at the call-chain deadline, WO-WZ-007). Distinct from a real + // fault: the unit of work is retryable in a fresh transaction. Emitted by + // the cms dbexec side (adopted separately) and mapped guest-side to + // bnwasm.ErrTxExpired. + AbiErrorCode_ABI_ERROR_CODE_TX_EXPIRED AbiErrorCode = 6 ) // Enum value maps for AbiErrorCode. @@ -142,6 +148,7 @@ var ( 3: "ABI_ERROR_CODE_UNIMPLEMENTED", 4: "ABI_ERROR_CODE_DEADLINE_EXCEEDED", 5: "ABI_ERROR_CODE_PERMISSION_DENIED", + 6: "ABI_ERROR_CODE_TX_EXPIRED", } AbiErrorCode_value = map[string]int32{ "ABI_ERROR_CODE_UNSPECIFIED": 0, @@ -150,6 +157,7 @@ var ( "ABI_ERROR_CODE_UNIMPLEMENTED": 3, "ABI_ERROR_CODE_DEADLINE_EXCEEDED": 4, "ABI_ERROR_CODE_PERMISSION_DENIED": 5, + "ABI_ERROR_CODE_TX_EXPIRED": 6, } ) @@ -1293,14 +1301,15 @@ const file_v1_invoke_proto_rawDesc = "" + "\vHOOK_UNLOAD\x10\x06\x12\x12\n" + "\x0eHOOK_RAG_FETCH\x10\a\x12\x13\n" + "\x0fHOOK_MEDIA_HOOK\x10\b\x12\x11\n" + - "\rHOOK_DESCRIBE\x10\t*\xd4\x01\n" + + "\rHOOK_DESCRIBE\x10\t*\xf3\x01\n" + "\fAbiErrorCode\x12\x1e\n" + "\x1aABI_ERROR_CODE_UNSPECIFIED\x10\x00\x12\x1b\n" + "\x17ABI_ERROR_CODE_INTERNAL\x10\x01\x12\x19\n" + "\x15ABI_ERROR_CODE_DECODE\x10\x02\x12 \n" + "\x1cABI_ERROR_CODE_UNIMPLEMENTED\x10\x03\x12$\n" + " ABI_ERROR_CODE_DEADLINE_EXCEEDED\x10\x04\x12$\n" + - " ABI_ERROR_CODE_PERMISSION_DENIED\x10\x05B0Z.git.dev.alexdunmow.com/block/core/abi/v1;abiv1b\x06proto3" + " ABI_ERROR_CODE_PERMISSION_DENIED\x10\x05\x12\x1d\n" + + "\x19ABI_ERROR_CODE_TX_EXPIRED\x10\x06B0Z.git.dev.alexdunmow.com/block/core/abi/v1;abiv1b\x06proto3" var ( file_v1_invoke_proto_rawDescOnce sync.Once diff --git a/abi/v1/manifest.pb.go b/abi/v1/manifest.pb.go index e00a8ea..c4f2bca 100644 --- a/abi/v1/manifest.pb.go +++ b/abi/v1/manifest.pb.go @@ -98,8 +98,14 @@ type PluginManifest struct { // (CoreServiceBindings.Bind becomes a static declaration; the host // constructs and mounts the handlers). CoreServiceBindings []*CoreServiceBinding `protobuf:"bytes,29,rep,name=core_service_bindings,json=coreServiceBindings,proto3" json:"core_service_bindings,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // data_dir requests a persistent per-plugin /data preopen at load. Its + // source of truth is the plugin.mod `data_dir = true` key, which lives + // outside the guest code, so DESCRIBE cannot populate it: the packer + // (`ninja plugin build`) stamps it into the manifest from plugin.mod so + // the loader reads one source. OFF by default. + DataDir bool `protobuf:"varint,30,opt,name=data_dir,json=dataDir,proto3" json:"data_dir,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *PluginManifest) Reset() { @@ -335,6 +341,13 @@ func (x *PluginManifest) GetCoreServiceBindings() []*CoreServiceBinding { return nil } +func (x *PluginManifest) GetDataDir() bool { + if x != nil { + return x.DataDir + } + return false +} + // Dependency mirrors plugin.Dependency. type Dependency struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1247,7 +1260,7 @@ var File_v1_manifest_proto protoreflect.FileDescriptor const file_v1_manifest_proto_rawDesc = "" + "\n" + - "\x11v1/manifest.proto\x12\x06abi.v1\"\x87\f\n" + + "\x11v1/manifest.proto\x12\x06abi.v1\"\xa2\f\n" + "\x0ePluginManifest\x12\x1f\n" + "\vabi_version\x18\x01 \x01(\rR\n" + "abiVersion\x12\x12\n" + @@ -1281,7 +1294,8 @@ const file_v1_manifest_proto_rawDesc = "" + "\x0fhas_unload_hook\x18\x1a \x01(\bR\rhasUnloadHook\x12&\n" + "\x0fhas_media_hooks\x18\x1b \x01(\bR\rhasMediaHooks\x12'\n" + "\x0fhas_provisioner\x18\x1c \x01(\bR\x0ehasProvisioner\x12N\n" + - "\x15core_service_bindings\x18\x1d \x03(\v2\x1a.abi.v1.CoreServiceBindingR\x13coreServiceBindings\x1aB\n" + + "\x15core_service_bindings\x18\x1d \x03(\v2\x1a.abi.v1.CoreServiceBindingR\x13coreServiceBindings\x12\x19\n" + + "\bdata_dir\x18\x1e \x01(\bR\adataDir\x1aB\n" + "\x14RbacMethodRolesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"a\n" + diff --git a/cmd/ninja/cmd/plugin.go b/cmd/ninja/cmd/plugin.go index 99a8dcd..8979e30 100644 --- a/cmd/ninja/cmd/plugin.go +++ b/cmd/ninja/cmd/plugin.go @@ -27,6 +27,8 @@ func newPluginCmd() *cobra.Command { c := &cobra.Command{Use: "plugin", Short: "Manage BlockNinja plugins"} c.AddCommand( newPluginInitCmd(), + newPluginBuildCmd(), + newPluginVerifyCmd(), newPluginPublishCmd(), newPluginStatusCmd(), newPluginListCmd(), @@ -1092,6 +1094,9 @@ func writeMod(path string, m *core.ModFile) error { if m.Plugin.Private { b.WriteString("private = true\n") } + if m.Plugin.DataDir { + b.WriteString("data_dir = true\n") + } if m.Compatibility != nil { b.WriteString("\n[compatibility]\n") fmt.Fprintf(&b, "block_core = %q\n", m.Compatibility.BlockCore) diff --git a/cmd/ninja/cmd/plugin_build.go b/cmd/ninja/cmd/plugin_build.go new file mode 100644 index 0000000..c83e1bd --- /dev/null +++ b/cmd/ninja/cmd/plugin_build.go @@ -0,0 +1,119 @@ +package cmd + +import ( + "context" + "fmt" + "strings" + + "github.com/spf13/cobra" + + "git.dev.alexdunmow.com/block/core/cmd/ninja/internal/bnp" +) + +func newPluginBuildCmd() *cobra.Command { + var dir, output string + cmd := &cobra.Command{ + Use: "build", + Short: "Compile a plugin to wasm and pack a .bnp artifact", + Long: `Compile the plugin in --dir to reactor-mode wasip1 wasm, extract its +static manifest by instantiating the module once (HOOK_DESCRIBE), and pack a +.bnp (tar.zst of plugin.wasm, plugin.mod, manifest.pb, plus migrations/, +schemas/, assets/, web/dist when present). + +No Docker or podman: the whole pipeline is the local Go toolchain (>= 1.24) +plus wazero. This replaces the in-container .so compile.`, + RunE: func(c *cobra.Command, _ []string) error { + res, err := bnp.Build(context.Background(), bnp.BuildOptions{Dir: dir, Output: output}) + if err != nil { + return err + } + printBuildSummary(res) + return nil + }, + } + cmd.Flags().StringVar(&dir, "dir", ".", "Plugin repo directory") + cmd.Flags().StringVarP(&output, "output", "o", "", "Output .bnp path (default -.bnp)") + return cmd +} + +func newPluginVerifyCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "verify ", + Short: "Validate a .bnp against the loader's layout/name/abi/path/size checks", + Long: `Re-run the CMS reader's checks on a .bnp standalone so CI and the registry +can gate uploads: required members present, path-safety and size caps on +extraction, manifest decodes, abi_version supported, and manifest name matches +plugin.mod. Prints a named reason for each malformed class.`, + Args: cobra.ExactArgs(1), + RunE: func(c *cobra.Command, args []string) error { + res, err := bnp.Verify(args[0]) + if err != nil { + return err + } + printVerifySummary(args[0], res) + return nil + }, + } + return cmd +} + +func printBuildSummary(r *bnp.BuildResult) { + hooks := "(none)" + if len(r.Hooks) > 0 { + hooks = strings.Join(r.Hooks, ", ") + } + dirs := "(none)" + if len(r.IncludedDirs) > 0 { + dirs = strings.Join(r.IncludedDirs, ", ") + } + fmt.Printf("Built %s@%s → %s\n", r.Name, r.Version, r.OutputPath) + fmt.Println(" ┌───────────────────────────────────────────") + fmt.Printf(" │ artifact size %s (%s uncompressed)\n", humanBytes(r.ArtifactBytes), humanBytes(r.UncompBytes)) + fmt.Printf(" │ plugin.wasm %s\n", humanBytes(r.WasmBytes)) + fmt.Printf(" │ manifest.pb %s\n", humanBytes(r.ManifestBytes)) + fmt.Printf(" │ blocks %d\n", r.BlockCount) + fmt.Printf(" │ templates %d\n", r.TemplateCount) + fmt.Printf(" │ admin pages %d\n", r.AdminPages) + fmt.Printf(" │ job types %d\n", r.JobTypes) + fmt.Printf(" │ hooks %s\n", hooks) + fmt.Printf(" │ bundled dirs %s\n", dirs) + fmt.Printf(" │ data_dir grant %t\n", r.DataDir) + fmt.Println(" └───────────────────────────────────────────") +} + +func printVerifySummary(path string, r *bnp.VerifyResult) { + var dirs []string + if r.HasMigrations { + dirs = append(dirs, "migrations") + } + if r.HasSchemas { + dirs = append(dirs, "schemas") + } + if r.HasAssets { + dirs = append(dirs, "assets") + } + if r.HasWeb { + dirs = append(dirs, "web") + } + joined := "(none)" + if len(dirs) > 0 { + joined = strings.Join(dirs, ", ") + } + fmt.Printf("OK: %s\n", path) + fmt.Printf(" name=%s version=%s abi_version=%d blocks=%d data_dir=%t dirs=[%s]\n", + r.Name, r.Version, r.ABIVersion, r.BlockCount, r.DataDir, joined) +} + +// humanBytes formats a byte count with a binary unit suffix. +func humanBytes(n int64) string { + const unit = 1024 + if n < unit { + return fmt.Sprintf("%d B", n) + } + div, exp := int64(unit), 0 + for x := n / unit; x >= unit; x /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp]) +} diff --git a/cmd/ninja/cmd/plugin_build_test.go b/cmd/ninja/cmd/plugin_build_test.go new file mode 100644 index 0000000..9f85f12 --- /dev/null +++ b/cmd/ninja/cmd/plugin_build_test.go @@ -0,0 +1,279 @@ +package cmd + +import ( + "archive/tar" + "context" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + abiv1 "git.dev.alexdunmow.com/block/core/abi/v1" + "git.dev.alexdunmow.com/block/core/cmd/ninja/internal/bnp" + "github.com/klauspost/compress/zstd" + "google.golang.org/protobuf/proto" +) + +// fixtureDir resolves a testdata plugin fixture in the core module tree. +func fixtureDir(t *testing.T, name string) string { + t.Helper() + dir, err := filepath.Abs(filepath.Join("..", "..", "..", "plugin", "wasmguest", "testdata", name)) + if err != nil { + t.Fatalf("resolve fixture %s: %v", name, err) + } + if _, err := os.Stat(filepath.Join(dir, "plugin.mod")); err != nil { + t.Fatalf("fixture %s missing plugin.mod: %v", name, err) + } + return dir +} + +// TestPluginBuildAndVerify is the WO-WZ-009 acceptance e2e: build the WZ-002 +// fixture repo → a .bnp exists, `verify` passes, and its manifest decodes with +// the expected block keys and the data_dir grant from plugin.mod. +func TestPluginBuildAndVerify(t *testing.T) { + if testing.Short() { + t.Skip("compiles a wasm module; skipped in -short") + } + out := filepath.Join(t.TempDir(), "wasmfixture-0.0.1.bnp") + + res, err := bnp.Build(context.Background(), bnp.BuildOptions{ + Dir: fixtureDir(t, "fixture"), + Output: out, + }) + if err != nil { + t.Fatalf("build: %v", err) + } + if res.Name != "wasmfixture" || res.Version != "0.0.1" { + t.Errorf("identity = %q/%q", res.Name, res.Version) + } + if res.BlockCount != 1 || res.TemplateCount != 1 || res.AdminPages != 1 { + t.Errorf("counts blocks=%d templates=%d admin=%d", res.BlockCount, res.TemplateCount, res.AdminPages) + } + if !res.DataDir { + t.Errorf("data_dir grant not carried from plugin.mod into the manifest") + } + if !slices.Contains(res.Hooks, "load") { + t.Errorf("hooks = %v, want load present", res.Hooks) + } + if _, err := os.Stat(out); err != nil { + t.Fatalf("artifact not written: %v", err) + } + + // verify passes on the produced artifact. + vr, err := bnp.Verify(out) + if err != nil { + t.Fatalf("verify: %v", err) + } + if vr.Name != "wasmfixture" || vr.ABIVersion != 1 || !vr.DataDir { + t.Errorf("verify result = %+v", vr) + } + + // manifest decodes with expected block keys. + m, err := bnp.ReadManifest(out) + if err != nil { + t.Fatalf("read manifest: %v", err) + } + var keys []string + for _, b := range m.GetBlocks() { + keys = append(keys, b.GetKey()) + } + if !slices.Contains(keys, "wasmfixture:greeting") { + t.Errorf("block keys = %v, want wasmfixture:greeting", keys) + } + if !slices.Contains(m.GetTemplateKeys(), "wasmfixture-page") { + t.Errorf("template keys = %v, want wasmfixture-page", m.GetTemplateKeys()) + } + if !m.GetDataDir() { + t.Errorf("manifest.data_dir = false, want true") + } +} + +// TestPluginBuildDefaultOutputName confirms the default artifact name is +// -.bnp in the working directory. +func TestPluginBuildDefaultOutputName(t *testing.T) { + if testing.Short() { + t.Skip("compiles a wasm module; skipped in -short") + } + dir := fixtureDir(t, "fixture") // resolve to an absolute path before chdir + wd, _ := os.Getwd() + t.Cleanup(func() { _ = os.Chdir(wd) }) + tmp := t.TempDir() + if err := os.Chdir(tmp); err != nil { + t.Fatalf("chdir: %v", err) + } + res, err := bnp.Build(context.Background(), bnp.BuildOptions{Dir: dir}) + if err != nil { + t.Fatalf("build: %v", err) + } + if res.OutputPath != "wasmfixture-0.0.1.bnp" { + t.Errorf("default output = %q", res.OutputPath) + } + if _, err := os.Stat(filepath.Join(tmp, "wasmfixture-0.0.1.bnp")); err != nil { + t.Errorf("default artifact not in cwd: %v", err) + } +} + +// TestPluginBuildCapabilityInRegisterFails proves a plugin that reaches a host +// capability at describe time fails with an actionable error naming the call. +func TestPluginBuildCapabilityInRegisterFails(t *testing.T) { + if testing.Short() { + t.Skip("compiles a wasm module; skipped in -short") + } + out := filepath.Join(t.TempDir(), "capfixture.bnp") + _, err := bnp.Build(context.Background(), bnp.BuildOptions{ + Dir: fixtureDir(t, "capfixture"), + Output: out, + }) + if err == nil { + t.Fatal("expected build to fail on a describe-time capability call") + } + msg := err.Error() + if !strings.Contains(msg, "db.query") || !strings.Contains(msg, "manifest extraction") { + t.Errorf("error not actionable: %q (want it to name db.query + manifest extraction)", msg) + } + if _, statErr := os.Stat(out); statErr == nil { + t.Errorf("a .bnp must not be written when the build fails") + } +} + +// --- verify rejection classes (fast; no wasm compile) --- + +type fakeEntry struct { + name string + data []byte + typeflag byte +} + +// writeTarZst crafts an arbitrary tar.zst for the malformed-artifact tests. +func writeTarZst(t *testing.T, entries []fakeEntry) string { + t.Helper() + path := filepath.Join(t.TempDir(), "artifact.bnp") + f, err := os.Create(path) + if err != nil { + t.Fatalf("create: %v", err) + } + defer func() { _ = f.Close() }() + enc, _ := zstd.NewWriter(f) + tw := tar.NewWriter(enc) + for _, e := range entries { + typ := e.typeflag + if typ == 0 { + typ = tar.TypeReg + } + hdr := &tar.Header{Name: e.name, Typeflag: typ, Mode: 0o644, Size: int64(len(e.data))} + if typ == tar.TypeSymlink { + hdr.Linkname = string(e.data) + hdr.Size = 0 + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatalf("hdr %q: %v", e.name, err) + } + if typ == tar.TypeReg { + if _, err := tw.Write(e.data); err != nil { + t.Fatalf("write %q: %v", e.name, err) + } + } + } + _ = tw.Close() + _ = enc.Close() + return path +} + +func manifestBytes(t *testing.T, name string, abi uint32) []byte { + t.Helper() + b, err := proto.Marshal(&abiv1.PluginManifest{Name: name, Version: "1.0.0", AbiVersion: abi}) + if err != nil { + t.Fatalf("marshal manifest: %v", err) + } + return b +} + +func TestVerifyRejectsMalformed(t *testing.T) { + validMod := []byte("[plugin]\nname = \"demo\"\nversion = \"1.0.0\"\n") + + cases := []struct { + name string + entries []fakeEntry + wantMsg string + }{ + { + name: "missing manifest.pb", + entries: []fakeEntry{ + {name: "plugin.wasm", data: []byte("\x00asm")}, + {name: "plugin.mod", data: validMod}, + }, + wantMsg: "missing required manifest.pb", + }, + { + name: "undecodable manifest", + entries: []fakeEntry{ + {name: "plugin.wasm", data: []byte("\x00asm")}, + {name: "plugin.mod", data: validMod}, + {name: "manifest.pb", data: []byte("not-a-proto\xff\xff")}, + }, + wantMsg: "decode manifest.pb", + }, + { + name: "unsupported abi_version", + entries: []fakeEntry{ + {name: "plugin.wasm", data: []byte("\x00asm")}, + {name: "plugin.mod", data: validMod}, + {name: "manifest.pb", data: manifestBytes(t, "demo", 2)}, + }, + wantMsg: "unsupported abi_version 2", + }, + { + name: "empty manifest name", + entries: []fakeEntry{ + {name: "plugin.wasm", data: []byte("\x00asm")}, + {name: "plugin.mod", data: validMod}, + {name: "manifest.pb", data: manifestBytes(t, "", 1)}, + }, + wantMsg: "empty name", + }, + { + name: "name mismatch", + entries: []fakeEntry{ + {name: "plugin.wasm", data: []byte("\x00asm")}, + {name: "plugin.mod", data: validMod}, + {name: "manifest.pb", data: manifestBytes(t, "other", 1)}, + }, + wantMsg: "!= plugin.mod name", + }, + { + name: "absolute path entry", + entries: []fakeEntry{ + {name: "/etc/passwd", data: []byte("x")}, + }, + wantMsg: "absolute entry path", + }, + { + name: "traversal entry", + entries: []fakeEntry{ + {name: "../escape.txt", data: []byte("x")}, + }, + wantMsg: "traversal segment", + }, + { + name: "symlink entry", + entries: []fakeEntry{ + {name: "link", data: []byte("/etc/passwd"), typeflag: tar.TypeSymlink}, + }, + wantMsg: "non-regular entry", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + path := writeTarZst(t, tc.entries) + _, err := bnp.Verify(path) + if err == nil { + t.Fatalf("expected rejection, got nil") + } + if !strings.Contains(err.Error(), tc.wantMsg) { + t.Errorf("error %q does not mention %q", err.Error(), tc.wantMsg) + } + }) + } +} diff --git a/cmd/ninja/internal/bnp/build.go b/cmd/ninja/internal/bnp/build.go new file mode 100644 index 0000000..2e2094b --- /dev/null +++ b/cmd/ninja/internal/bnp/build.go @@ -0,0 +1,248 @@ +package bnp + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "sort" + "strconv" + + abiv1 "git.dev.alexdunmow.com/block/core/abi/v1" + core "git.dev.alexdunmow.com/block/core/plugin" + "google.golang.org/protobuf/proto" +) + +// Required top-level artifact members (mirrors the CMS reader). +const ( + fileWasm = "plugin.wasm" + fileMod = "plugin.mod" + fileManifest = "manifest.pb" +) + +// minGoMajor/minGoMinor is the lowest Go toolchain that can emit reactor-mode +// (`-buildmode=c-shared`) wasip1 modules the guest shim relies on. +const ( + minGoMajor = 1 + minGoMinor = 24 +) + +// BuildOptions configures Build. +type BuildOptions struct { + // Dir is the plugin repo root (holds plugin.mod and the main Go package). + Dir string + // Output is the destination .bnp path. Empty → "-.bnp" in + // the current working directory. + Output string +} + +// BuildResult summarizes a produced artifact for the CLI summary table. +type BuildResult struct { + OutputPath string + Name string + Version string + WasmBytes int64 + ManifestBytes int64 + ArtifactBytes int64 // on-disk .bnp size + UncompBytes int64 // sum of packed file sizes + BlockCount int + TemplateCount int + AdminPages int + JobTypes int + Hooks []string + DataDir bool + IncludedDirs []string +} + +// Build compiles the plugin in opts.Dir to wasm, extracts its manifest, and +// packs a .bnp. No Docker/podman: the whole pipeline is the local Go +// toolchain + wazero + tar.zst. +func Build(ctx context.Context, opts BuildOptions) (*BuildResult, error) { + dir := opts.Dir + if dir == "" { + dir = "." + } + dir, err := filepath.Abs(dir) + if err != nil { + return nil, fmt.Errorf("resolve dir: %w", err) + } + + modPath := filepath.Join(dir, fileMod) + modBytes, err := os.ReadFile(modPath) + if err != nil { + return nil, fmt.Errorf("read plugin.mod (is %s a plugin repo?): %w", dir, err) + } + mod, err := core.ParseModFull(modBytes) + if err != nil { + return nil, fmt.Errorf("parse plugin.mod: %w", err) + } + if mod.Plugin.Name == "" || mod.Plugin.Version == "" { + return nil, fmt.Errorf("plugin.mod must set both name and version") + } + + if err := checkGoVersion(ctx); err != nil { + return nil, err + } + + // 1. Compile to reactor-mode wasip1 c-shared. + tmp, err := os.MkdirTemp("", "ninja-build-*") + if err != nil { + return nil, fmt.Errorf("temp dir: %w", err) + } + defer func() { _ = os.RemoveAll(tmp) }() + wasmPath := filepath.Join(tmp, fileWasm) + if err := buildWasm(ctx, dir, wasmPath); err != nil { + return nil, err + } + + // 2. Extract the manifest by driving DESCRIBE. + wasm, err := readWasm(wasmPath) + if err != nil { + return nil, err + } + manifest, err := ExtractManifest(ctx, wasm) + if err != nil { + return nil, fmt.Errorf("extract manifest: %w", err) + } + + // 3. Validate against plugin.mod before packing an incoherent artifact. + if manifest.GetName() != mod.Plugin.Name { + return nil, fmt.Errorf("manifest name %q != plugin.mod name %q", manifest.GetName(), mod.Plugin.Name) + } + if v := manifest.GetAbiVersion(); v != hostAbiVersion { + return nil, fmt.Errorf("manifest abi_version %d unsupported (packer speaks %d)", v, hostAbiVersion) + } + + // 4. Stamp the data_dir grant from plugin.mod. DESCRIBE cannot see + // plugin.mod (it lives outside guest code), so the packer is where the + // grant crosses from mod → manifest; the loader then reads one source. + manifest.DataDir = mod.Plugin.DataDir + + manifestBytes, err := proto.Marshal(manifest) + if err != nil { + return nil, fmt.Errorf("marshal manifest.pb: %w", err) + } + + // 5. Assemble artifact entries. + entries := []packEntry{ + {ArtifactPath: fileWasm, Source: wasmPath}, + {ArtifactPath: fileMod, Source: modPath}, + {ArtifactPath: fileManifest, Data: manifestBytes}, + } + var includedDirs []string + // dir-name → source subpath. web ships the Module Federation build output + // (web/dist) flattened under web/ so the reader's dirExists("web") fires. + optional := []struct{ artifact, src string }{ + {"migrations", "migrations"}, + {"schemas", "schemas"}, + {"assets", "assets"}, + {"web", filepath.Join("web", "dist")}, + } + for _, o := range optional { + dirEntries, has, dErr := collectDir(filepath.Join(dir, o.src), o.artifact) + if dErr != nil { + return nil, dErr + } + if has { + entries = append(entries, dirEntries...) + includedDirs = append(includedDirs, o.artifact) + } + } + + // 6. Pack. + outPath := opts.Output + if outPath == "" { + outPath = fmt.Sprintf("%s-%s.bnp", mod.Plugin.Name, mod.Plugin.Version) + } + uncomp, err := packArtifact(outPath, entries) + if err != nil { + return nil, err + } + + artInfo, err := os.Stat(outPath) + if err != nil { + return nil, fmt.Errorf("stat artifact: %w", err) + } + wasmInfo, _ := os.Stat(wasmPath) + + return &BuildResult{ + OutputPath: outPath, + Name: manifest.GetName(), + Version: manifest.GetVersion(), + WasmBytes: wasmInfo.Size(), + ManifestBytes: int64(len(manifestBytes)), + ArtifactBytes: artInfo.Size(), + UncompBytes: uncomp, + BlockCount: len(manifest.GetBlocks()), + TemplateCount: len(manifest.GetTemplateKeys()), + AdminPages: len(manifest.GetAdminPages()), + JobTypes: len(manifest.GetJobTypes()), + Hooks: hooksPresent(manifest), + DataDir: manifest.GetDataDir(), + IncludedDirs: includedDirs, + }, nil +} + +// buildWasm runs the reactor-mode wasip1 c-shared build in dir. +func buildWasm(ctx context.Context, dir, outPath string) error { + cmd := exec.CommandContext(ctx, "go", "build", "-buildmode=c-shared", "-o", outPath, ".") + cmd.Dir = dir + cmd.Env = append(os.Environ(), "GOOS=wasip1", "GOARCH=wasm") + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("go build (GOOS=wasip1 GOARCH=wasm -buildmode=c-shared) failed: %w\n%s", err, stderr.String()) + } + return nil +} + +var goVersionRe = regexp.MustCompile(`go(\d+)\.(\d+)(?:\.\d+)?`) + +// checkGoVersion enforces the minimum toolchain (Go 1.24) with a clear error. +func checkGoVersion(ctx context.Context) error { + out, err := exec.CommandContext(ctx, "go", "version").Output() + if err != nil { + return fmt.Errorf("`go version` failed (is the Go toolchain on PATH?): %w", err) + } + m := goVersionRe.FindStringSubmatch(string(out)) + if m == nil { + return fmt.Errorf("could not parse Go version from %q", string(out)) + } + major, _ := strconv.Atoi(m[1]) + minor, _ := strconv.Atoi(m[2]) + if major < minGoMajor || (major == minGoMajor && minor < minGoMinor) { + return fmt.Errorf( + "Go %d.%d+ is required to build reactor-mode wasip1 plugins; found %d.%d — upgrade your toolchain", + minGoMajor, minGoMinor, major, minor) + } + return nil +} + +// hooksPresent returns the sorted set of runtime hooks the manifest declares, +// for the summary table. +func hooksPresent(m *abiv1.PluginManifest) []string { + var hooks []string + if m.GetHasLoadHook() { + hooks = append(hooks, "load") + } + if m.GetHasUnloadHook() { + hooks = append(hooks, "unload") + } + if m.GetHasHttpHandler() { + hooks = append(hooks, "http") + } + if m.GetHasMediaHooks() { + hooks = append(hooks, "media") + } + if len(m.GetJobTypes()) > 0 { + hooks = append(hooks, "job") + } + if len(m.GetRagContentFetcherTypes()) > 0 { + hooks = append(hooks, "rag") + } + sort.Strings(hooks) + return hooks +} diff --git a/cmd/ninja/internal/bnp/manifest.go b/cmd/ninja/internal/bnp/manifest.go new file mode 100644 index 0000000..306ca86 --- /dev/null +++ b/cmd/ninja/internal/bnp/manifest.go @@ -0,0 +1,171 @@ +// Package bnp implements `ninja plugin build` and `ninja plugin verify`: it +// compiles a plugin to reactor-mode wasm, extracts its static manifest by +// instantiating the module once and calling HOOK_DESCRIBE, packs the .bnp +// artifact (tar.zst), and re-runs the CMS reader's layout/name/abi/path/size +// checks standalone. +// +// The verify rules here are a deliberate, documented duplication of the CMS +// reader (cms backend/plugin/bnp/reader.go, WO-WZ-008): core cannot import the +// CMS module, so publishers get the same gate the loader applies. WO-WZ-010's +// integration suite keeps the two in lockstep. +package bnp + +import ( + "context" + "fmt" + "os" + + abiv1 "git.dev.alexdunmow.com/block/core/abi/v1" + "github.com/tetratelabs/wazero" + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1" + "google.golang.org/protobuf/proto" +) + +// hostAbiVersion is the ABI major version the packer speaks when driving +// DESCRIBE. It must equal core's guest AbiVersion / the reader's supported +// major (1) or the guest's symmetric version gate rejects the call. +const hostAbiVersion uint32 = 1 + +// ExtractManifest instantiates a reactor-mode plugin.wasm with wazero, drives +// one HOOK_DESCRIBE round trip, and returns the decoded PluginManifest. +// +// The host functions the guest imports (blockninja.host_call) are stubbed to +// FAIL: DESCRIBE is publish-time and must not need a live host. A plugin that +// reaches a host capability while its manifest is being extracted (e.g. a +// db.* call from Register) gets an actionable error naming the offending +// method rather than a mystery hang or nil deref. +func ExtractManifest(ctx context.Context, wasm []byte) (*abiv1.PluginManifest, error) { + r := wazero.NewRuntime(ctx) + defer func() { _ = r.Close(ctx) }() + + if _, err := wasi_snapshot_preview1.Instantiate(ctx, r); err != nil { + return nil, fmt.Errorf("instantiate wasi: %w", err) + } + if err := installFailingHost(ctx, r); err != nil { + return nil, fmt.Errorf("install host stub: %w", err) + } + + mod, err := r.InstantiateWithConfig(ctx, wasm, + wazero.NewModuleConfig().WithStartFunctions("_initialize").WithName("plugin")) + if err != nil { + return nil, fmt.Errorf("instantiate module (is it a reactor-mode wasip1 c-shared build?): %w", err) + } + for _, export := range []string{"bn_alloc", "bn_invoke", "bn_free"} { + if mod.ExportedFunction(export) == nil { + return nil, fmt.Errorf("module does not export %s — not a BlockNinja wasm guest (missing wasmguest.Serve boilerplate?)", export) + } + } + + resp, err := invokeDescribe(ctx, mod) + if err != nil { + return nil, err + } + if e := resp.GetError(); e != nil { + return nil, fmt.Errorf("DESCRIBE failed (%s): %s", e.GetCode(), e.GetMessage()) + } + dr := &abiv1.DescribeResponse{} + if err := proto.Unmarshal(resp.GetPayload(), dr); err != nil { + return nil, fmt.Errorf("decode DescribeResponse: %w", err) + } + m := dr.GetManifest() + if m == nil { + return nil, fmt.Errorf("DESCRIBE returned an empty manifest") + } + return m, nil +} + +// invokeDescribe drives one bn_alloc / bn_invoke round trip for HOOK_DESCRIBE +// through guest linear memory, mirroring the calling convention in +// core/docs/wasm-abi.md. +func invokeDescribe(ctx context.Context, mod api.Module) (*abiv1.InvokeResponse, error) { + payload, err := proto.Marshal(&abiv1.DescribeRequest{HostAbiVersion: hostAbiVersion}) + if err != nil { + return nil, fmt.Errorf("marshal DescribeRequest: %w", err) + } + req, err := proto.Marshal(&abiv1.InvokeRequest{Hook: abiv1.Hook_HOOK_DESCRIBE, Payload: payload}) + if err != nil { + return nil, fmt.Errorf("marshal InvokeRequest: %w", err) + } + + res, err := mod.ExportedFunction("bn_alloc").Call(ctx, uint64(len(req))) + if err != nil { + return nil, fmt.Errorf("bn_alloc: %w", err) + } + ptr := uint32(res[0]) + if ptr == 0 { + return nil, fmt.Errorf("bn_alloc returned 0") + } + if !mod.Memory().Write(ptr, req) { + return nil, fmt.Errorf("write DESCRIBE request out of range") + } + + res, err = mod.ExportedFunction("bn_invoke").Call(ctx, + uint64(abiv1.Hook_HOOK_DESCRIBE), uint64(ptr), uint64(len(req))) + if err != nil { + return nil, fmt.Errorf("bn_invoke(DESCRIBE): %w", err) + } + packed := res[0] + if packed == 0 { + return nil, fmt.Errorf("bn_invoke returned packed 0 (guest could not produce a response envelope)") + } + respBytes, ok := mod.Memory().Read(uint32(packed>>32), uint32(packed)) + if !ok { + return nil, fmt.Errorf("read DESCRIBE response out of range") + } + resp := &abiv1.InvokeResponse{} + if err := proto.Unmarshal(respBytes, resp); err != nil { + return nil, fmt.Errorf("decode InvokeResponse: %w", err) + } + _, _ = mod.ExportedFunction("bn_free").Call(ctx, uint64(ptr)) + return resp, nil +} + +// installFailingHost exports blockninja.host_call so the guest module can be +// instantiated, but answers every capability call with a PERMISSION_DENIED +// AbiError naming the method — DESCRIBE must not depend on the host. +func installFailingHost(ctx context.Context, r wazero.Runtime) error { + _, err := r.NewHostModuleBuilder("blockninja"). + NewFunctionBuilder(). + WithFunc(func(ctx context.Context, mod api.Module, ptr, size uint32) uint64 { + method := "" + if data, ok := mod.Memory().Read(ptr, size); ok { + hcr := &abiv1.HostCallRequest{} + if proto.Unmarshal(data, hcr) == nil && hcr.GetMethod() != "" { + method = hcr.GetMethod() + } + } + out, err := proto.Marshal(&abiv1.HostCallResponse{ + Error: &abiv1.AbiError{ + Code: abiv1.AbiErrorCode_ABI_ERROR_CODE_PERMISSION_DENIED, + Message: fmt.Sprintf( + "capability %q is unavailable during manifest extraction (DESCRIBE): a plugin must not call host capabilities at register/describe time", + method), + }, + }) + if err != nil { + return 0 + } + res, err := mod.ExportedFunction("bn_alloc").Call(ctx, uint64(len(out))) + if err != nil || len(res) == 0 || res[0] == 0 { + return 0 + } + respPtr := uint32(res[0]) + if !mod.Memory().Write(respPtr, out) { + return 0 + } + return uint64(respPtr)<<32 | uint64(uint32(len(out))) + }). + Export("host_call"). + Instantiate(ctx) + return err +} + +// readWasm is a tiny helper so callers can pass a path. +func readWasm(path string) ([]byte, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read wasm %s: %w", path, err) + } + return b, nil +} diff --git a/cmd/ninja/internal/bnp/pack.go b/cmd/ninja/internal/bnp/pack.go new file mode 100644 index 0000000..589559e --- /dev/null +++ b/cmd/ninja/internal/bnp/pack.go @@ -0,0 +1,138 @@ +package bnp + +import ( + "archive/tar" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/klauspost/compress/zstd" +) + +// packEntry is one file destined for the artifact: its path inside the .bnp +// (always forward-slash relative) and either literal Data or a Source file to +// stream from. +type packEntry struct { + ArtifactPath string + Data []byte + Source string +} + +// packArtifact writes entries as a zstd-compressed tar to outPath. Entries are +// sorted by artifact path for a deterministic archive. Directory entries are +// synthesized as needed so the reader's dirExists checks fire for +// migrations/, schemas/, assets/, web/. +func packArtifact(outPath string, entries []packEntry) (int64, error) { + sort.Slice(entries, func(i, j int) bool { return entries[i].ArtifactPath < entries[j].ArtifactPath }) + + out, err := os.Create(outPath) + if err != nil { + return 0, fmt.Errorf("create %s: %w", outPath, err) + } + defer func() { _ = out.Close() }() + + enc, err := zstd.NewWriter(out, zstd.WithEncoderLevel(zstd.SpeedDefault)) + if err != nil { + return 0, err + } + tw := tar.NewWriter(enc) + + seenDir := map[string]bool{} + writeDir := func(dir string) error { + if dir == "" || dir == "." || seenDir[dir] { + return nil + } + seenDir[dir] = true + return tw.WriteHeader(&tar.Header{ + Name: dir + "/", + Typeflag: tar.TypeDir, + Mode: 0o755, + }) + } + + var total int64 + for _, e := range entries { + name := filepath.ToSlash(e.ArtifactPath) + // Ensure parent directories exist as explicit entries. + for _, d := range parentDirs(name) { + if err := writeDir(d); err != nil { + return 0, fmt.Errorf("write dir header %q: %w", d, err) + } + } + + var data []byte + if e.Source != "" { + data, err = os.ReadFile(e.Source) + if err != nil { + return 0, fmt.Errorf("read %s: %w", e.Source, err) + } + } else { + data = e.Data + } + if err := tw.WriteHeader(&tar.Header{ + Name: name, + Typeflag: tar.TypeReg, + Mode: 0o644, + Size: int64(len(data)), + }); err != nil { + return 0, fmt.Errorf("write header %q: %w", name, err) + } + if _, err := tw.Write(data); err != nil { + return 0, fmt.Errorf("write %q: %w", name, err) + } + total += int64(len(data)) + } + + if err := tw.Close(); err != nil { + return 0, fmt.Errorf("close tar: %w", err) + } + if err := enc.Close(); err != nil { + return 0, fmt.Errorf("close zstd: %w", err) + } + return total, nil +} + +// parentDirs returns the ancestor directories of a forward-slash path, from +// shallowest to deepest ("a/b/c.txt" → ["a", "a/b"]). +func parentDirs(name string) []string { + var dirs []string + parts := strings.Split(name, "/") + for i := 1; i < len(parts); i++ { + dirs = append(dirs, strings.Join(parts[:i], "/")) + } + return dirs +} + +// collectDir returns pack entries for every regular file under root, mapped +// under artifactPrefix. Symlinks and irregular files are skipped. Returns +// (entries, hadFiles). +func collectDir(root, artifactPrefix string) ([]packEntry, bool, error) { + info, err := os.Stat(root) + if err != nil || !info.IsDir() { + return nil, false, nil //nolint:nilerr // absent optional dir is not an error + } + var entries []packEntry + walkErr := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !d.Type().IsRegular() { + return nil + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + entries = append(entries, packEntry{ + ArtifactPath: artifactPrefix + "/" + filepath.ToSlash(rel), + Source: path, + }) + return nil + }) + if walkErr != nil { + return nil, false, fmt.Errorf("walk %s: %w", root, walkErr) + } + return entries, len(entries) > 0, nil +} diff --git a/cmd/ninja/internal/bnp/verify.go b/cmd/ninja/internal/bnp/verify.go new file mode 100644 index 0000000..cb47516 --- /dev/null +++ b/cmd/ninja/internal/bnp/verify.go @@ -0,0 +1,275 @@ +package bnp + +import ( + "archive/tar" + "bufio" + "bytes" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + abiv1 "git.dev.alexdunmow.com/block/core/abi/v1" + "github.com/klauspost/compress/zstd" + "google.golang.org/protobuf/proto" +) + +// These constants and the validation flow below are a DELIBERATE duplication +// of the CMS reader (cms backend/plugin/bnp/reader.go, WO-WZ-008). core cannot +// import the CMS module, so `ninja plugin verify` re-implements the same gate a +// publisher's artifact will meet at load time. WO-WZ-010's integration suite +// keeps the two copies in lockstep; change both together. +const ( + // supportedABIMajor is the ABI major version a host understands. + supportedABIMajor uint32 = 1 + // maxArtifactBytes caps total decompressed size (matches the registry + // publish cap, 1 GiB) to bound a decompression bomb. + maxArtifactBytes int64 = 1 << 30 + // maxEntryBytes caps a single extracted file. + maxEntryBytes int64 = 512 * 1024 * 1024 +) + +// VerifyResult reports what a valid artifact contains. +type VerifyResult struct { + Name string + Version string + ABIVersion uint32 + DataDir bool + BlockCount int + HasMigrations bool + HasSchemas bool + HasAssets bool + HasWeb bool +} + +// Verify extracts bnpPath into a temp dir and applies the reader's checks: +// path-safety + size caps on extraction, required members present, manifest +// decodes, abi major supported, non-empty name, and manifest name == plugin.mod +// name. The temp dir is always cleaned up. A named error is returned for each +// malformed class. +func Verify(bnpPath string) (*VerifyResult, error) { + destDir, err := os.MkdirTemp("", "ninja-verify-*") + if err != nil { + return nil, fmt.Errorf("bnp: temp dir: %w", err) + } + defer func() { _ = os.RemoveAll(destDir) }() + + f, err := os.Open(bnpPath) + if err != nil { + return nil, fmt.Errorf("bnp: open %s: %w", bnpPath, err) + } + defer func() { _ = f.Close() }() + + if err := extractTarZst(f, destDir); err != nil { + return nil, err + } + + for _, req := range []string{fileWasm, fileMod, fileManifest} { + if !fileRegular(filepath.Join(destDir, req)) { + return nil, fmt.Errorf("bnp: artifact missing required %s", req) + } + } + + manifestBytes, err := os.ReadFile(filepath.Join(destDir, fileManifest)) + if err != nil { + return nil, fmt.Errorf("bnp: read manifest.pb: %w", err) + } + manifest := &abiv1.PluginManifest{} + if err := proto.Unmarshal(manifestBytes, manifest); err != nil { + return nil, fmt.Errorf("bnp: decode manifest.pb: %w", err) + } + if v := manifest.GetAbiVersion(); v != supportedABIMajor { + return nil, fmt.Errorf("bnp: unsupported abi_version %d (host supports %d)", v, supportedABIMajor) + } + name := manifest.GetName() + if name == "" { + return nil, errors.New("bnp: manifest has empty name") + } + + modBytes, err := os.ReadFile(filepath.Join(destDir, fileMod)) + if err != nil { + return nil, fmt.Errorf("bnp: read plugin.mod: %w", err) + } + modName := parseModName(modBytes) + if modName == "" { + return nil, errors.New("bnp: plugin.mod has no name") + } + if modName != name { + return nil, fmt.Errorf("bnp: manifest name %q != plugin.mod name %q", name, modName) + } + + return &VerifyResult{ + Name: name, + Version: manifest.GetVersion(), + ABIVersion: manifest.GetAbiVersion(), + DataDir: manifest.GetDataDir(), + BlockCount: len(manifest.GetBlocks()), + HasMigrations: dirExists(filepath.Join(destDir, "migrations")), + HasSchemas: dirExists(filepath.Join(destDir, "schemas")), + HasAssets: dirExists(filepath.Join(destDir, "assets")), + HasWeb: dirExists(filepath.Join(destDir, "web")), + }, nil +} + +// ReadManifest extracts a .bnp into a temp dir (with the same path-safety and +// size caps as Verify) and returns its decoded manifest.pb. It does not enforce +// the name-match / abi checks — use Verify for the full gate. +func ReadManifest(bnpPath string) (*abiv1.PluginManifest, error) { + destDir, err := os.MkdirTemp("", "ninja-manifest-*") + if err != nil { + return nil, fmt.Errorf("bnp: temp dir: %w", err) + } + defer func() { _ = os.RemoveAll(destDir) }() + + f, err := os.Open(bnpPath) + if err != nil { + return nil, fmt.Errorf("bnp: open %s: %w", bnpPath, err) + } + defer func() { _ = f.Close() }() + if err := extractTarZst(f, destDir); err != nil { + return nil, err + } + b, err := os.ReadFile(filepath.Join(destDir, fileManifest)) + if err != nil { + return nil, fmt.Errorf("bnp: read manifest.pb: %w", err) + } + m := &abiv1.PluginManifest{} + if err := proto.Unmarshal(b, m); err != nil { + return nil, fmt.Errorf("bnp: decode manifest.pb: %w", err) + } + return m, nil +} + +// extractTarZst streams a zstd-compressed tar into destDir, rejecting +// non-regular entries, absolute/traversal paths, and enforcing size caps. +func extractTarZst(r io.Reader, destDir string) error { + zr, err := zstd.NewReader(r) + if err != nil { + return fmt.Errorf("bnp: open zstd stream: %w", err) + } + defer zr.Close() + + tr := tar.NewReader(zr) + var total int64 + root, err := filepath.Abs(destDir) + if err != nil { + return fmt.Errorf("bnp: resolve dest: %w", err) + } + + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return fmt.Errorf("bnp: read tar: %w", err) + } + + switch hdr.Typeflag { + case tar.TypeReg, tar.TypeDir: + default: + return fmt.Errorf("bnp: rejected non-regular entry %q (type %d)", hdr.Name, hdr.Typeflag) + } + + target, err := safeJoin(root, hdr.Name) + if err != nil { + return err + } + + if hdr.Typeflag == tar.TypeDir { + if err := os.MkdirAll(target, 0o755); err != nil { + return fmt.Errorf("bnp: mkdir %q: %w", hdr.Name, err) + } + continue + } + + if hdr.Size > maxEntryBytes { + return fmt.Errorf("bnp: entry %q exceeds per-file cap (%d > %d)", hdr.Name, hdr.Size, maxEntryBytes) + } + if total+hdr.Size > maxArtifactBytes { + return fmt.Errorf("bnp: artifact exceeds total size cap (%d)", maxArtifactBytes) + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return fmt.Errorf("bnp: mkdir for %q: %w", hdr.Name, err) + } + n, err := writeExtractedFile(target, io.LimitReader(tr, maxArtifactBytes-total)) + if err != nil { + return fmt.Errorf("bnp: extract %q: %w", hdr.Name, err) + } + total += n + if total > maxArtifactBytes { + return fmt.Errorf("bnp: artifact exceeds total size cap (%d)", maxArtifactBytes) + } + } + return nil +} + +// safeJoin joins a relative tar entry onto root, rejecting absolute paths and +// traversal segments. +func safeJoin(root, name string) (string, error) { + norm := strings.ReplaceAll(name, `\`, "/") + if norm == "" || norm == "." { + return "", fmt.Errorf("bnp: empty entry name %q", name) + } + if filepath.IsAbs(name) || strings.HasPrefix(norm, "/") { + return "", fmt.Errorf("bnp: absolute entry path %q rejected", name) + } + for _, seg := range strings.Split(norm, "/") { + if seg == ".." { + return "", fmt.Errorf("bnp: entry %q contains a traversal segment", name) + } + } + target := filepath.Join(root, filepath.Clean(norm)) + rel, err := filepath.Rel(root, target) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("bnp: entry %q escapes artifact root", name) + } + return target, nil +} + +func writeExtractedFile(path string, r io.Reader) (int64, error) { + out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return 0, err + } + n, err := io.Copy(out, r) + closeErr := out.Close() + if err != nil { + return n, err + } + return n, closeErr +} + +func fileRegular(p string) bool { + fi, err := os.Stat(p) + return err == nil && fi.Mode().IsRegular() +} + +func dirExists(p string) bool { + fi, err := os.Stat(p) + return err == nil && fi.IsDir() +} + +// parseModName extracts the `name = "..."` value from a plugin.mod body, +// mirroring the reader's tolerant line scan (works whether or not the key sits +// under a [plugin] table). +func parseModName(data []byte) string { + sc := bufio.NewScanner(bytes.NewReader(data)) + for sc.Scan() { + after, ok := strings.CutPrefix(strings.TrimSpace(sc.Text()), "name") + if !ok { + continue + } + val, ok := strings.CutPrefix(strings.TrimSpace(after), "=") + if !ok { + continue + } + val = strings.Trim(strings.TrimSpace(val), `"`) + if val != "" { + return val + } + } + return "" +} diff --git a/docs/wasm-abi.md b/docs/wasm-abi.md index ab15403..21b59ca 100644 --- a/docs/wasm-abi.md +++ b/docs/wasm-abi.md @@ -61,6 +61,65 @@ init funcs — hence `Serve`, which stores the registration and returns. > the module — either way the exports are never callable. The original > blocking-main design was amended to reactor mode for this reason. +## Building & packing (`ninja plugin build`) + +`ninja plugin build` turns a plugin repo into a `.bnp` artifact in one command — +no Docker/podman, just the local Go toolchain (**≥ 1.24**, enforced with a clear +error) plus wazero. It replaces the in-container `.so` compile and the old +`make build-so`. Steps: + +1. Parse `plugin.mod` for `name`/`version` (both required). +2. `GOOS=wasip1 GOARCH=wasm go build -buildmode=c-shared -o plugin.wasm .` + (reactor mode). +3. Extract `manifest.pb`: instantiate `plugin.wasm` with wazero and drive one + `HOOK_DESCRIBE`. The host functions are **stubbed to fail** — DESCRIBE must + not need a live host, so a plugin that reaches a capability at + register/describe time (e.g. a `db.*` call from `Register`) gets an + actionable error **naming the offending method** instead of a hang. +4. Stamp `manifest.data_dir` from `plugin.mod` (see below). +5. Pack a `tar.zst`: `plugin.wasm`, `plugin.mod`, `manifest.pb`, plus + `migrations/`, `schemas/`, `assets/`, and `web/dist` (Module Federation + output, flattened under `web/`) when present. + +``` +ninja plugin build [--dir .] [-o -.bnp] +ninja plugin verify +``` + +`ninja plugin verify` re-runs the CMS `.bnp` reader's checks standalone (layout, +required members, path-safety + size caps on extraction, `abi_version` support, +and manifest name == `plugin.mod` name) so CI and the registry can gate uploads. +These rules are a **deliberate duplication** of the reader +(`cms backend/plugin/bnp/reader.go`); core cannot import the CMS module, and +WO-WZ-010's integration suite keeps the two in lockstep. + +### Makefile convention — `make build-wasm` + +Plugin repos expose a `build-wasm` target (replacing `build-so`) that just calls +the CLI: + +```make +.PHONY: build-wasm +build-wasm: + ninja plugin build +``` + +### `plugin.mod` `data_dir` + +`plugin.mod` may set an optional first-class boolean: + +```toml +[plugin] +name = "my-plugin" +version = "0.1.0" +data_dir = true # request a persistent per-plugin /data preopen at load +``` + +`data_dir` is a real field on the mod parser (not an arbitrary key) precisely so +the CLI's mod round-trip cannot silently drop it — `writeMod` reconstructs +`plugin.mod` from known struct fields only. `ninja plugin build` copies it into +`PluginManifest.data_dir`; OFF by default. + ## Calling convention (ptr+len, packed u64) wasm exports can only pass `i32/i64/f32/f64`, so all payloads cross as @@ -248,6 +307,12 @@ zero value / best-effort on transport failure. | `ABI_ERROR_CODE_UNIMPLEMENTED` | Callee does not implement the hook/capability (e.g. host too old for a new capability method). | None. | | `ABI_ERROR_CODE_DEADLINE_EXCEEDED` | `deadline_ms` elapsed. | Instance considered poisoned, discarded. | | `ABI_ERROR_CODE_PERMISSION_DENIED` | Caller not entitled to the capability. | None. | +| `ABI_ERROR_CODE_TX_EXPIRED` | A `db.*` call named a transaction handle the host already expired (dropped at the call-chain deadline, WO-WZ-007). | None — retryable. The guest maps it to `bnwasm.ErrTxExpired`; plugin code re-runs the unit of work in a fresh transaction. | + +`ABI_ERROR_CODE_TX_EXPIRED` is emitted by the cms `dbexec` side (adopted +separately from this WO) in place of the earlier INTERNAL+message-marker +workaround, so guests can distinguish a retryable tx expiry from a real fault +via `errors.Is(err, bnwasm.ErrTxExpired)`. Host-function errors surface to plugin code as ordinary Go errors via the guest SDK. Repeated instance failures trip the existing @@ -287,6 +352,7 @@ Field-by-field: | `Unload` | `has_unload_hook` + `HOOK_UNLOAD` | | `Migrations` | `.bnp` artifact `migrations/` directory (Goose runs host-side; never crosses) | | `RequiredIconPacks` | `required_icon_packs` | +| *(none — `plugin.mod` `data_dir`)* | `data_dir` (bool). Not a `PluginRegistration` field: the grant lives in `plugin.mod`, which the guest code cannot see, so `ninja plugin build` stamps `PluginManifest.data_dir` from `plugin.mod` after DESCRIBE. The `.bnp` reader also reads it straight from `plugin.mod` as a fallback. | ## Render context diff --git a/plugin/mod.go b/plugin/mod.go index 7f986e4..9a4e035 100644 --- a/plugin/mod.go +++ b/plugin/mod.go @@ -45,6 +45,14 @@ type ModPlugin struct { // field, and the publish flow attributes the plugin to the publisher's // active account rather than to a public scope. Private bool `toml:"private,omitempty"` + // DataDir requests a persistent per-plugin /data preopen at load time. It + // is a first-class field (not an arbitrary key) precisely so the CLI's + // mod round-trip cannot silently drop it: writeMod reconstructs plugin.mod + // from struct fields only, so any key without a home here is lost on the + // next `ninja plugin init`/bump. `ninja plugin build` stamps this into the + // packed manifest (PluginManifest.data_dir); the .bnp reader also reads it + // straight from plugin.mod as a fallback. OFF by default. + DataDir bool `toml:"data_dir,omitempty"` } type ModCompat struct { diff --git a/plugin/mod_test.go b/plugin/mod_test.go index a324a6f..7eaa9ce 100644 --- a/plugin/mod_test.go +++ b/plugin/mod_test.go @@ -132,6 +132,37 @@ private = true } } +func TestParseModFull_DataDirField(t *testing.T) { + src := []byte(` +[plugin] +name = "stateful-plugin" +version = "0.1.0" +data_dir = true +`) + m, err := ParseModFull(src) + if err != nil { + t.Fatalf("ParseModFull err: %v", err) + } + if !m.Plugin.DataDir { + t.Errorf("DataDir = false, want true") + } +} + +func TestParseModFull_DataDirDefaultsFalse(t *testing.T) { + src := []byte(` +[plugin] +name = "stateless-plugin" +version = "0.1.0" +`) + m, err := ParseModFull(src) + if err != nil { + t.Fatalf("ParseModFull err: %v", err) + } + if m.Plugin.DataDir { + t.Errorf("DataDir = true, want false (absent key)") + } +} + func TestParseModFull_PrivateDefaultsFalse(t *testing.T) { src := []byte(` [plugin] diff --git a/plugin/wasmguest/bnwasm/transport.go b/plugin/wasmguest/bnwasm/transport.go index da08039..e42e26b 100644 --- a/plugin/wasmguest/bnwasm/transport.go +++ b/plugin/wasmguest/bnwasm/transport.go @@ -62,6 +62,36 @@ func dbErr(e *abiv1.DbError) error { return &pgconn.PgError{Code: e.GetCode(), Message: e.GetMessage()} } +// ErrTxExpired is returned by a db.* call whose transaction handle the host +// has already expired — it drops the handle at the call-chain deadline +// (WO-WZ-007) so a guest can never pin a connection. Unlike a real fault this +// is retryable: re-run the unit of work in a fresh transaction. The host +// signals it with ABI_ERROR_CODE_TX_EXPIRED (cms dbexec side, adopted +// separately); the transport wraps ErrTxExpired around the raw host error so +// callers can `errors.Is(err, bnwasm.ErrTxExpired)`. +var ErrTxExpired = errors.New("bnwasm: transaction expired; retry in a new transaction") + +// abiCoded is satisfied by *wasmguest.HostError (its AbiErrorCode method) +// without importing that wasip1-only package, so bnwasm can classify a +// transport error by its ABI code across the sandbox boundary. +type abiCoded interface { + AbiErrorCode() abiv1.AbiErrorCode +} + +// mapTransportErr translates a transport-level ABI error into a bnwasm +// sentinel where one exists (TX_EXPIRED → ErrTxExpired), leaving every other +// error untouched. Applied to the raw t(...) error at each db.* call site. +func mapTransportErr(err error) error { + if err == nil { + return nil + } + var coded abiCoded + if errors.As(err, &coded) && coded.AbiErrorCode() == abiv1.AbiErrorCode_ABI_ERROR_CODE_TX_EXPIRED { + return fmt.Errorf("%w: %v", ErrTxExpired, err) + } + return err +} + func (t Transport) query(ctx context.Context, sql string, txHandle uint64, args []any) (*abiv1.DbRowsResponse, error) { if t == nil { return nil, errNoHost @@ -75,7 +105,7 @@ func (t Transport) query(ctx context.Context, sql string, txHandle uint64, args } resp := &abiv1.DbRowsResponse{} if err := t(methodQuery, &abiv1.DbQueryRequest{Sql: sql, Args: vals, TxHandle: txHandle}, resp); err != nil { - return nil, err + return nil, mapTransportErr(err) } if err := dbErr(resp.GetError()); err != nil { return nil, err @@ -96,7 +126,7 @@ func (t Transport) exec(ctx context.Context, sql string, txHandle uint64, args [ } resp := &abiv1.DbExecResponse{} if err := t(methodExec, &abiv1.DbExecRequest{Sql: sql, Args: vals, TxHandle: txHandle}, resp); err != nil { - return pgconn.CommandTag{}, err + return pgconn.CommandTag{}, mapTransportErr(err) } if err := dbErr(resp.GetError()); err != nil { return pgconn.CommandTag{}, err @@ -115,7 +145,7 @@ func (t Transport) txBegin(ctx context.Context) (uint64, error) { } resp := &abiv1.DbTxBeginResponse{} if err := t(methodTxBegin, &abiv1.DbTxBeginRequest{}, resp); err != nil { - return 0, err + return 0, mapTransportErr(err) } if err := dbErr(resp.GetError()); err != nil { return 0, err @@ -135,7 +165,7 @@ func (t Transport) txCommit(ctx context.Context, handle uint64) error { } resp := &abiv1.DbTxCommitResponse{} if err := t(methodTxCommit, &abiv1.DbTxCommitRequest{TxHandle: handle}, resp); err != nil { - return err + return mapTransportErr(err) } return dbErr(resp.GetError()) } @@ -149,7 +179,7 @@ func (t Transport) txRollback(ctx context.Context, handle uint64) error { } resp := &abiv1.DbTxRollbackResponse{} if err := t(methodTxRollback, &abiv1.DbTxRollbackRequest{TxHandle: handle}, resp); err != nil { - return err + return mapTransportErr(err) } return dbErr(resp.GetError()) } diff --git a/plugin/wasmguest/bnwasm/tx_expired_test.go b/plugin/wasmguest/bnwasm/tx_expired_test.go new file mode 100644 index 0000000..a83a24f --- /dev/null +++ b/plugin/wasmguest/bnwasm/tx_expired_test.go @@ -0,0 +1,74 @@ +package bnwasm + +import ( + "context" + "errors" + "testing" + + abiv1 "git.dev.alexdunmow.com/block/core/abi/v1" + "google.golang.org/protobuf/proto" +) + +// fakeHostErr mirrors *wasmguest.HostError's AbiErrorCode() method so the +// transport's cross-boundary classification (mapTransportErr) can be exercised +// natively, without importing the wasip1-only wasmguest package. +type fakeHostErr struct { + code abiv1.AbiErrorCode + msg string +} + +func (e *fakeHostErr) Error() string { return e.msg } +func (e *fakeHostErr) AbiErrorCode() abiv1.AbiErrorCode { return e.code } + +// transportReturning builds a Transport whose every call fails with err. +func transportReturning(err error) Transport { + return func(_ string, _, _ proto.Message) error { return err } +} + +func TestTransportMapsTxExpiredToSentinel(t *testing.T) { + hostErr := &fakeHostErr{code: abiv1.AbiErrorCode_ABI_ERROR_CODE_TX_EXPIRED, msg: "host: tx handle expired"} + tr := transportReturning(hostErr) + ctx := context.Background() + + // Every db.* verb that crosses the transport must classify TX_EXPIRED. + t.Run("query", func(t *testing.T) { + _, err := tr.query(ctx, "SELECT 1", 7, nil) + if !errors.Is(err, ErrTxExpired) { + t.Fatalf("query err = %v, want ErrTxExpired", err) + } + }) + t.Run("exec", func(t *testing.T) { + _, err := tr.exec(ctx, "DELETE FROM t", 7, nil) + if !errors.Is(err, ErrTxExpired) { + t.Fatalf("exec err = %v, want ErrTxExpired", err) + } + }) + t.Run("txBegin", func(t *testing.T) { + _, err := tr.txBegin(ctx) + if !errors.Is(err, ErrTxExpired) { + t.Fatalf("txBegin err = %v, want ErrTxExpired", err) + } + }) + t.Run("txCommit", func(t *testing.T) { + if err := tr.txCommit(ctx, 7); !errors.Is(err, ErrTxExpired) { + t.Fatalf("txCommit err = %v, want ErrTxExpired", err) + } + }) + t.Run("txRollback", func(t *testing.T) { + if err := tr.txRollback(ctx, 7); !errors.Is(err, ErrTxExpired) { + t.Fatalf("txRollback err = %v, want ErrTxExpired", err) + } + }) +} + +func TestTransportLeavesOtherAbiErrorsUntouched(t *testing.T) { + hostErr := &fakeHostErr{code: abiv1.AbiErrorCode_ABI_ERROR_CODE_INTERNAL, msg: "boom"} + tr := transportReturning(hostErr) + _, err := tr.query(context.Background(), "SELECT 1", 0, nil) + if errors.Is(err, ErrTxExpired) { + t.Fatalf("non-tx-expired error was misclassified as ErrTxExpired: %v", err) + } + if !errors.Is(err, error(hostErr)) { + t.Fatalf("original host error not preserved: %v", err) + } +} diff --git a/plugin/wasmguest/testdata/capfixture/main.go b/plugin/wasmguest/testdata/capfixture/main.go new file mode 100644 index 0000000..c1cf042 --- /dev/null +++ b/plugin/wasmguest/testdata/capfixture/main.go @@ -0,0 +1,8 @@ +//go:build wasip1 + +package main + +import "git.dev.alexdunmow.com/block/core/plugin/wasmguest" + +func init() { wasmguest.Serve(Registration) } +func main() {} // never called — reactor mode diff --git a/plugin/wasmguest/testdata/capfixture/plugin.mod b/plugin/wasmguest/testdata/capfixture/plugin.mod new file mode 100644 index 0000000..2bb3a67 --- /dev/null +++ b/plugin/wasmguest/testdata/capfixture/plugin.mod @@ -0,0 +1,6 @@ +[plugin] +name = "capfixture" +scope = "@blockninja" +version = "0.0.1" +description = "Negative fixture: Register hits a host capability at describe time" +kind = "plugin" diff --git a/plugin/wasmguest/testdata/capfixture/registration.go b/plugin/wasmguest/testdata/capfixture/registration.go new file mode 100644 index 0000000..d680db3 --- /dev/null +++ b/plugin/wasmguest/testdata/capfixture/registration.go @@ -0,0 +1,38 @@ +// Package main is a NEGATIVE build fixture: its Register hook reaches a host +// capability (a db.* call) at describe time, which is illegal — DESCRIBE runs +// with no live host. `ninja plugin build` must surface an actionable error +// naming the offending call rather than hanging or crashing. +package main + +import ( + "context" + "database/sql" + "fmt" + + // Registers the "bnwasm" database/sql driver (its init runs sql.Register). + _ "git.dev.alexdunmow.com/block/core/plugin/wasmguest/bnwasm" + + "git.dev.alexdunmow.com/block/core/blocks" + "git.dev.alexdunmow.com/block/core/plugin" + "git.dev.alexdunmow.com/block/core/templates" +) + +var Registration = plugin.PluginRegistration{ + Name: "capfixture", + Version: "0.0.1", + Register: func(_ templates.TemplateRegistry, _ blocks.BlockRegistry) error { + // Illegal describe-time capability use: query the host DB from + // Register. Over the packer's failing host stub this returns an error + // naming "db.query". + db, err := sql.Open("bnwasm", "") + if err != nil { + return err + } + defer func() { _ = db.Close() }() + var n int + if err := db.QueryRowContext(context.Background(), "SELECT 1").Scan(&n); err != nil { + return fmt.Errorf("describe-time db probe: %w", err) + } + return nil + }, +} diff --git a/plugin/wasmguest/testdata/fixture/plugin.mod b/plugin/wasmguest/testdata/fixture/plugin.mod new file mode 100644 index 0000000..4c8f7cd --- /dev/null +++ b/plugin/wasmguest/testdata/fixture/plugin.mod @@ -0,0 +1,8 @@ +[plugin] +name = "wasmfixture" +display_name = "Wasm Fixture" +scope = "@blockninja" +version = "0.0.1" +description = "WO-WZ-002 acceptance fixture: one block, one template, one admin page" +kind = "plugin" +data_dir = true