From 62b245d24fa24431cf0d149cbb503c016e4aa4cd Mon Sep 17 00:00:00 2001 From: Alex Dunmow Date: Sun, 5 Jul 2026 08:57:31 +0800 Subject: [PATCH] feat(backup): shared backup archive format v3 (writer/reader/crypto) for cms + platform backups Co-Authored-By: Claude Fable 5 --- backup/archive.go | 224 +++++++++++++++++++++++++++++++++++++++++ backup/archive_test.go | 97 ++++++++++++++++++ backup/crypto.go | 120 ++++++++++++++++++++++ backup/manifest.go | 74 ++++++++++++++ go.mod | 1 + go.sum | 2 + 6 files changed, 518 insertions(+) create mode 100644 backup/archive.go create mode 100644 backup/archive_test.go create mode 100644 backup/crypto.go create mode 100644 backup/manifest.go diff --git a/backup/archive.go b/backup/archive.go new file mode 100644 index 0000000..80f540d --- /dev/null +++ b/backup/archive.go @@ -0,0 +1,224 @@ +package backup + +import ( + "archive/zip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "time" +) + +// Section maps a zip path prefix to an on-disk directory to be archived. +type Section struct { + Prefix string + Dir string +} + +// BuildInput is the full set of inputs for BuildArchive. +type BuildInput struct { + Source Source + Dump []byte + Password string + Sections []Section + // Extra holds additional fixed zip entries (zip path -> contents), written + // after database.dump and before section files. + Extra map[string][]byte +} + +// backupFile pairs a zip path with its source path on disk. +type backupFile struct { + zipPath string + diskPath string +} + +// collectSectionFiles walks a section directory and returns zipPath→diskPath +// pairs. A missing directory is skipped silently (count 0, no warning), matching +// the cms collectSectionFiles semantics. Files that cannot be accessed inside an +// existing directory append warnings and are skipped. +func collectSectionFiles(ctx context.Context, section Section, warnings *[]string) ([]backupFile, error) { + if section.Dir == "" { + return nil, nil + } + if _, err := os.Stat(section.Dir); os.IsNotExist(err) { + return nil, nil + } + + var files []backupFile + err := filepath.Walk(section.Dir, func(path string, info os.FileInfo, err error) error { + if ctx.Err() != nil { + return ctx.Err() + } + if err != nil { + *warnings = append(*warnings, fmt.Sprintf("Failed to access %s: %v", path, err)) + return nil + } + if info.IsDir() || !info.Mode().IsRegular() { + return nil + } + + relPath, err := filepath.Rel(section.Dir, path) + if err != nil { + *warnings = append(*warnings, fmt.Sprintf("Failed to get relative path for %s: %v", path, err)) + return nil + } + + files = append(files, backupFile{ + zipPath: section.Prefix + filepath.ToSlash(relPath), + diskPath: path, + }) + return nil + }) + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + *warnings = append(*warnings, fmt.Sprintf("Failed to walk %s: %v", section.Dir, err)) + } + return files, nil +} + +// BuildArchive streams a backup zip archive to w: a manifest.json, the +// password-encrypted database dump, any Extra entries, then the files of each +// configured section in order. It returns the manifest and any non-fatal +// warnings encountered while collecting/writing section files. +// +// Zip entry ordering is: manifest.json, database.dump, Extra (sorted by path), +// then section files in section order. Counts for the well-known section +// prefixes (media/, brand/, icons/, styles/, plugins/) are recorded in the +// manifest. +func BuildArchive(ctx context.Context, w io.Writer, in BuildInput) (*Manifest, []string, error) { + var warnings []string + + if err := ctx.Err(); err != nil { + return nil, warnings, err + } + + // Checksum of the unencrypted dump. + sum := sha256.Sum256(in.Dump) + checksum := hex.EncodeToString(sum[:]) + + // Encrypt the dump. + encryptedDump, encParams, err := EncryptWithPassword(in.Dump, in.Password) + if err != nil { + return nil, warnings, fmt.Errorf("encryption failed: %w", err) + } + + // Collect files for every configured section, preserving section order. + sectionFiles := make([][]backupFile, len(in.Sections)) + countsByPrefix := make(map[string]int) + for i, section := range in.Sections { + files, err := collectSectionFiles(ctx, section, &warnings) + if err != nil { + return nil, warnings, err + } + sectionFiles[i] = files + countsByPrefix[section.Prefix] += len(files) + } + + manifest := &Manifest{ + Version: FormatVersion, + Format: FormatName, + ExportedAt: time.Now().UTC(), + Source: in.Source, + DatabaseSize: int64(len(in.Dump)), + MediaCount: countsByPrefix["media/"], + BrandCount: countsByPrefix["brand/"], + IconsCount: countsByPrefix["icons/"], + StylesCount: countsByPrefix["styles/"], + PluginsCount: countsByPrefix["plugins/"], + Encryption: encParams, + Checksum: checksum, + } + + manifestJSON, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return nil, warnings, fmt.Errorf("manifest marshal failed: %w", err) + } + + if err := ctx.Err(); err != nil { + return nil, warnings, err + } + + zipWriter := zip.NewWriter(w) + + manifestEntry, err := zipWriter.Create("manifest.json") + if err != nil { + return nil, warnings, fmt.Errorf("failed to create manifest.json in zip: %w", err) + } + if _, err := manifestEntry.Write(manifestJSON); err != nil { + return nil, warnings, fmt.Errorf("failed to write manifest.json: %w", err) + } + + dumpEntry, err := zipWriter.Create("database.dump") + if err != nil { + return nil, warnings, fmt.Errorf("failed to create database.dump in zip: %w", err) + } + if _, err := dumpEntry.Write(encryptedDump); err != nil { + return nil, warnings, fmt.Errorf("failed to write database.dump: %w", err) + } + + // Extra fixed entries, in a stable (sorted) order for reproducibility. + if len(in.Extra) > 0 { + names := make([]string, 0, len(in.Extra)) + for name := range in.Extra { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + entry, err := zipWriter.Create(name) + if err != nil { + return nil, warnings, fmt.Errorf("failed to create %s in zip: %w", name, err) + } + if _, err := entry.Write(in.Extra[name]); err != nil { + return nil, warnings, fmt.Errorf("failed to write %s: %w", name, err) + } + } + } + + // Section files, with periodic context checks (every 10 files). + written := 0 + for _, files := range sectionFiles { + for _, bf := range files { + if written%10 == 0 { + if err := ctx.Err(); err != nil { + return nil, warnings, err + } + } + written++ + + src, err := os.Open(bf.diskPath) + if err != nil { + warnings = append(warnings, fmt.Sprintf("Failed to read %s for zip: %v", bf.diskPath, err)) + continue + } + + entry, err := zipWriter.Create(bf.zipPath) + if err != nil { + closeBestEffort(src) + warnings = append(warnings, fmt.Sprintf("Failed to create %s in zip: %v", bf.zipPath, err)) + continue + } + + if _, err := io.Copy(entry, src); err != nil { + warnings = append(warnings, fmt.Sprintf("Failed to write %s to zip: %v", bf.zipPath, err)) + } + closeBestEffort(src) + } + } + + if err := zipWriter.Close(); err != nil { + return nil, warnings, fmt.Errorf("failed to close zip: %w", err) + } + + return manifest, warnings, nil +} + +func closeBestEffort(c io.Closer) { + _ = c.Close() +} diff --git a/backup/archive_test.go b/backup/archive_test.go new file mode 100644 index 0000000..6452dd5 --- /dev/null +++ b/backup/archive_test.go @@ -0,0 +1,97 @@ +package backup + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "io" + "os" + "path/filepath" + "testing" +) + +func readZipEntry(t *testing.T, data []byte, name string) []byte { + t.Helper() + zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + t.Fatalf("zip.NewReader: %v", err) + } + for _, f := range zr.File { + if f.Name == name { + rc, err := f.Open() + if err != nil { + t.Fatalf("open %s: %v", name, err) + } + defer rc.Close() + b, err := io.ReadAll(rc) + if err != nil { + t.Fatalf("read %s: %v", name, err) + } + return b + } + } + t.Fatalf("zip entry %q not found", name) + return nil +} + +func TestBuildArchiveRoundTrip(t *testing.T) { + dir := t.TempDir() + mediaDir := filepath.Join(dir, "media") + if err := os.MkdirAll(filepath.Join(mediaDir, "sub"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(mediaDir, "sub", "a.txt"), []byte("hello"), 0o644); err != nil { + t.Fatal(err) + } + + dump := []byte("fake pg_dump custom-format payload") + var buf bytes.Buffer + manifest, warnings, err := BuildArchive(context.Background(), &buf, BuildInput{ + Source: Source{SiteTitle: "Test Site", Domain: "test.example.com"}, + Dump: dump, + Password: "correct horse", + Sections: []Section{ + {Prefix: "media/", Dir: mediaDir}, + {Prefix: "brand/", Dir: filepath.Join(dir, "does-not-exist")}, // missing dir → warning, not error + }, + }) + if err != nil { + t.Fatalf("BuildArchive: %v", err) + } + if len(warnings) != 0 { + t.Fatalf("missing section dir must be silently skipped (cms parity), got warnings: %v", warnings) + } + if manifest.Version != FormatVersion || manifest.Format != FormatName { + t.Fatalf("manifest version/format = %d/%q", manifest.Version, manifest.Format) + } + if manifest.MediaCount != 1 || manifest.BrandCount != 0 { + t.Fatalf("counts: media=%d brand=%d", manifest.MediaCount, manifest.BrandCount) + } + wantSum := sha256.Sum256(dump) + if manifest.Checksum != hex.EncodeToString(wantSum[:]) { + t.Fatalf("checksum mismatch") + } + + // Read back: manifest parses, dump decrypts to the original bytes. + ra := bytes.NewReader(buf.Bytes()) + got, err := ReadManifest(ra, int64(buf.Len())) + if err != nil { + t.Fatalf("ReadManifest: %v", err) + } + encDump := readZipEntry(t, buf.Bytes(), "database.dump") + plain, err := DecryptWithPassword(encDump, "correct horse", got.Encryption) + if err != nil { + t.Fatalf("DecryptWithPassword: %v", err) + } + if !bytes.Equal(plain, dump) { + t.Fatal("decrypted dump != original") + } + if _, err := DecryptWithPassword(encDump, "wrong password", got.Encryption); err == nil { + t.Fatal("wrong password must fail") + } + if string(readZipEntry(t, buf.Bytes(), "media/sub/a.txt")) != "hello" { + t.Fatal("media file missing or corrupted in archive") + } +} diff --git a/backup/crypto.go b/backup/crypto.go new file mode 100644 index 0000000..4acf290 --- /dev/null +++ b/backup/crypto.go @@ -0,0 +1,120 @@ +package backup + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/base64" + "fmt" + "io" + + "golang.org/x/crypto/argon2" +) + +const ( + // Argon2id parameters — byte-identical to the cms settings_crypto so that + // archives produced here decrypt through the existing cms restore path. + argon2Time = 3 + argon2Memory = 64 * 1024 // 64 MB + argon2Threads = 4 + argon2KeyLen = 32 // AES-256 + saltLength = 16 +) + +// EncryptionParams stores the parameters used for key derivation. Its JSON shape +// is wire-identical to the cms EncryptionParams so manifests interoperate. +type EncryptionParams struct { + Algorithm string `json:"algorithm"` + KDF string `json:"kdf"` + Salt string `json:"salt"` + Iterations int `json:"iterations"` + Memory int `json:"memory"` + Parallelism int `json:"parallelism"` +} + +// deriveKey derives a 32-byte key from password using Argon2id. +func deriveKey(password string, salt []byte) []byte { + return argon2.IDKey([]byte(password), salt, argon2Time, argon2Memory, argon2Threads, argon2KeyLen) +} + +// EncryptWithPassword encrypts data with AES-256-GCM using a password-derived key. +// The nonce is prepended to the returned ciphertext. +func EncryptWithPassword(plaintext []byte, password string) ([]byte, *EncryptionParams, error) { + // Generate random salt + salt := make([]byte, saltLength) + if _, err := io.ReadFull(rand.Reader, salt); err != nil { + return nil, nil, fmt.Errorf("failed to generate salt: %w", err) + } + + // Derive key from password + key := deriveKey(password, salt) + + // Create AES cipher + block, err := aes.NewCipher(key) + if err != nil { + return nil, nil, fmt.Errorf("failed to create cipher: %w", err) + } + + // Create GCM mode + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, nil, fmt.Errorf("failed to create GCM: %w", err) + } + + // Generate nonce + nonce := make([]byte, gcm.NonceSize()) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + return nil, nil, fmt.Errorf("failed to generate nonce: %w", err) + } + + // Encrypt (prepend nonce to ciphertext) + ciphertext := gcm.Seal(nonce, nonce, plaintext, nil) + + return ciphertext, &EncryptionParams{ + Algorithm: "aes-256-gcm", + KDF: "argon2id", + Salt: base64.StdEncoding.EncodeToString(salt), + Iterations: argon2Time, + Memory: argon2Memory, + Parallelism: argon2Threads, + }, nil +} + +// DecryptWithPassword decrypts data encrypted with EncryptWithPassword. +func DecryptWithPassword(ciphertext []byte, password string, params *EncryptionParams) ([]byte, error) { + // Decode salt + salt, err := base64.StdEncoding.DecodeString(params.Salt) + if err != nil { + return nil, fmt.Errorf("failed to decode salt: %w", err) + } + + // Derive key from password + key := deriveKey(password, salt) + + // Create AES cipher + block, err := aes.NewCipher(key) + if err != nil { + return nil, fmt.Errorf("failed to create cipher: %w", err) + } + + // Create GCM mode + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, fmt.Errorf("failed to create GCM: %w", err) + } + + // Extract nonce + nonceSize := gcm.NonceSize() + if len(ciphertext) < nonceSize { + return nil, fmt.Errorf("ciphertext too short") + } + nonce, ciphertext := ciphertext[:nonceSize], ciphertext[nonceSize:] + + // Decrypt + plaintext, err := gcm.Open(nil, nonce, ciphertext, nil) + if err != nil { + return nil, fmt.Errorf("decryption failed - invalid password or corrupted data") + } + + return plaintext, nil +} diff --git a/backup/manifest.go b/backup/manifest.go new file mode 100644 index 0000000..a7594da --- /dev/null +++ b/backup/manifest.go @@ -0,0 +1,74 @@ +package backup + +import ( + "archive/zip" + "encoding/json" + "fmt" + "io" + "time" +) + +const ( + // FormatVersion / FormatName identify the backup archive format. They are + // byte-identical to the cms constants (backupFormatVersion / backupFormatName) + // so archives interoperate with the existing cms restore path. + FormatVersion = 3 + FormatName = "blockninja-pgdump-backup" +) + +// Source identifies where a backup came from. Field-for-field identical to the +// cms BackupSource JSON shape. +type Source struct { + SiteTitle string `json:"site_title"` + Domain string `json:"domain,omitempty"` +} + +// Manifest is the metadata stored in manifest.json. It is a field-for-field copy +// of the cms BackupManifest so archives built here restore through the existing +// cms path unchanged. +type Manifest struct { + Version int `json:"version"` + Format string `json:"format"` + ExportedAt time.Time `json:"exported_at"` + Source Source `json:"source"` + DatabaseSize int64 `json:"database_size"` + MediaCount int `json:"media_count"` + BrandCount int `json:"brand_count,omitempty"` + IconsCount int `json:"icons_count,omitempty"` + StylesCount int `json:"styles_count,omitempty"` + PluginsCount int `json:"plugins_count,omitempty"` + Encryption *EncryptionParams `json:"encryption,omitempty"` + Checksum string `json:"checksum"` // SHA-256 of unencrypted database dump +} + +// ReadManifest opens a backup zip archive and parses its manifest.json entry. +func ReadManifest(r io.ReaderAt, size int64) (*Manifest, error) { + zr, err := zip.NewReader(r, size) + if err != nil { + return nil, fmt.Errorf("failed to open backup archive: %w", err) + } + + for _, f := range zr.File { + if f.Name != "manifest.json" { + continue + } + rc, err := f.Open() + if err != nil { + return nil, fmt.Errorf("failed to open manifest.json: %w", err) + } + defer rc.Close() + + data, err := io.ReadAll(rc) + if err != nil { + return nil, fmt.Errorf("failed to read manifest.json: %w", err) + } + + var m Manifest + if err := json.Unmarshal(data, &m); err != nil { + return nil, fmt.Errorf("failed to parse manifest.json: %w", err) + } + return &m, nil + } + + return nil, fmt.Errorf("manifest.json not found in backup archive") +} diff --git a/go.mod b/go.mod index 2972a5b..2f3757d 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.9.2 github.com/tetratelabs/wazero v1.12.0 + golang.org/x/crypto v0.48.0 golang.org/x/mod v0.34.0 google.golang.org/protobuf v1.36.11 ) diff --git a/go.sum b/go.sum index 31f585f..cd8a5be 100644 --- a/go.sum +++ b/go.sum @@ -36,6 +36,8 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tetratelabs/wazero v1.12.0 h1:DuWcpNu/FzgEXgGBDp8J1Spc+CWOvvtvVyjKlaZopYU= github.com/tetratelabs/wazero v1.12.0/go.mod h1:LvKtzl2RqO4gyF27BiXU+nKAjcV8f38U+kP/q2vgxh0= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=