From 68b9cee88023eb6a9a7a8073b9af94f40211039c Mon Sep 17 00:00:00 2001 From: DonVoo Date: Fri, 17 Jul 2026 00:37:02 +0200 Subject: [PATCH] Add verified snapshots and harden archive identity --- INSTALL.md | 34 +- backend/02-database.go | 159 ++++++-- backend/04-imap-source.go | 39 +- backend/04-imap-source_test.go | 50 +++ backend/06-mbox.go | 45 ++- backend/06-mbox_test.go | 161 ++++++++ backend/14-snapshot.go | 711 +++++++++++++++++++++++++++++++++ backend/14-snapshot_test.go | 209 ++++++++++ main.go | 16 + 9 files changed, 1372 insertions(+), 52 deletions(-) create mode 100644 backend/14-snapshot.go create mode 100644 backend/14-snapshot_test.go diff --git a/INSTALL.md b/INSTALL.md index 972ad9e..c716802 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -47,8 +47,10 @@ Ohne Web, z. B. als Cronjob oder Dauerlauf im Umzugsfenster: ./mail-graveyard --run dr-gold --watch # ein Konto, Delta-Schleife bis Stop ``` -Dank Message-ID-Cache kopiert jeder Lauf nur Neues — beliebig oft wiederholbar -ohne Dubletten. Für den Dauerlauf reicht `nohup`/systemd/Aufgabenplanung. +Die kanonische Inhaltsidentitaet verhindert dabei doppelte **Schreibvorgaenge** +in Archiv und Ziel. Der aktuelle Volllauf laedt trotzdem jede Quellmail erneut, +bevor er deren Body-Hash pruefen kann. Grosse Konten deshalb bis zum +UID-Delta-Watcher nicht in kurzen Intervallen als Dauerlauf starten. ## 5. Wartung @@ -56,18 +58,46 @@ ohne Dubletten. Für den Dauerlauf reicht `nohup`/systemd/Aufgabenplanung. ./mail-graveyard --reindex all ./mail-graveyard --dedup-target konto-name # Dry-Run ./mail-graveyard --dedup-target konto-name --apply # loescht Dubletten im Ziel +./mail-graveyard --snapshot /sicherungen/stand-2026-07-16 +./mail-graveyard --verify-snapshot /sicherungen/stand-2026-07-16 ``` `--reindex` baut den Archiv-Index fuer vorhandene mbox/mbox.zst-Dateien neu auf. `--dedup-target` ist bewusst ein Dry-Run, bis `--apply` gesetzt wird; Mails ohne Message-ID werden nie geloescht. +`--snapshot` erzeugt zuerst per SQLite `VACUUM INTO` einen konsistenten +DB-Stand. Anschliessend kopiert es aus jeder append-only mbox exakt das durch +`MAX(file_offset+frame_len)` **dieses DB-Standes** belegte Praefix. Das Ziel +muss neu sein und enthaelt `data/mail-graveyard.db`, `backup/...` sowie ein +Manifest mit Grenzen und SHA-256-Pruefsummen. Ein Fehler hinterlaesst ein klar +benanntes `.partial-*`-Verzeichnis und wird nie als fertiger Stand umbenannt. +Die Datenbank wird wegen der enthaltenen Klartext-Zugangsdaten mit `0600` +geschrieben; die Verifikation weist unter Linux weiter gefasste Rechte zurueck. + +`--verify-snapshot` veraendert weder Produktion noch Snapshot: Es prueft +Manifest, Dateigroessen, SHA-256, SQLite-Integritaet, Stufenreihenfolge und den +Identitaetsdigest. Danach kopiert es nur die Snapshot-DB in ein temporaeres +Restore-Verzeichnis und reindiziert dort alle mbox-Dateien. Der +Identitaetsdigest muss vor und nach dem Reindex gleich bleiben. + +Der lokale Snapshot ist nur die konsistente **Vorstufe** eines Backups. Er +enthaelt Kontozugangsdaten und muss anschliessend verschluesselt auf einen +raeumlich getrennten Datentraeger uebertragen werden. + ## 6. Sicherheit Tool hält fremde IMAP-Passwörter und kann Mails senden → Default bind `127.0.0.1`. Falls remote nötig: hinter Caddy + `/vadmin`-Mail-2FA. `config.json` und `*.db` nie committen. +Die Live-SQLite-Datei sowie ihre WAL-/SHM-Sidecars werden bei jedem Start auf +`0600` gesetzt. Das gilt auch fuer weitere `*.db`, `*.db-wal` und `*.db-shm` +direkt im konfigurierten Datenverzeichnis sowie fuer die Datenbank eines neu +erzeugten Snapshots. Kann ein Dateisystem Unix-Rechte nicht abbilden (z. B. +bestimmte exFAT-/CIFS-Mounts), startet das Werkzeug bewusst nicht: Zugangsdaten +bleiben fail-closed statt unbemerkt lesbar. + ## 7. Reihenfolge bei einem echten Umzug 1. Ziel-Postfächer beim neuen Hoster anlegen (`archiv-…`, `temp-…`). diff --git a/backend/02-database.go b/backend/02-database.go index b972cdd..a37670a 100644 --- a/backend/02-database.go +++ b/backend/02-database.go @@ -6,6 +6,8 @@ import ( "errors" "fmt" "os" + "path/filepath" + "runtime" "strings" _ "modernc.org/sqlite" @@ -85,6 +87,10 @@ func ConnectDB(initDB bool) error { _ = db.Close() return err } + if err := secureSQLiteFiles(Cfg.DBPath); err != nil { + _ = db.Close() + return err + } schema := []string{ `CREATE TABLE IF NOT EXISTS accounts( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -361,7 +367,7 @@ func ensureDBFile(path string, initDB bool) error { if info.IsDir() { return fmt.Errorf("DB %q ist ein Verzeichnis", path) } - return nil + return chmodPrivateFile(path) } if !errors.Is(err, os.ErrNotExist) { return err @@ -373,7 +379,57 @@ func ensureDBFile(path string, initDB bool) error { if err != nil { return err } - return f.Close() + if err := f.Close(); err != nil { + return err + } + return chmodPrivateFile(path) +} + +func secureSQLiteFiles(path string) error { + directory := filepath.Dir(path) + entries, err := os.ReadDir(directory) + if err != nil { + return fmt.Errorf("scan SQLite directory %q: %w", directory, err) + } + for _, entry := range entries { + if entry.Type()&os.ModeSymlink != 0 || !isSQLiteDataFile(entry.Name()) { + continue + } + info, err := entry.Info() + if err != nil { + return fmt.Errorf("inspect SQLite file %q: %w", filepath.Join(directory, entry.Name()), err) + } + if !info.Mode().IsRegular() { + continue + } + candidate := filepath.Join(directory, entry.Name()) + if err := chmodPrivateFile(candidate); err != nil { + return fmt.Errorf("secure SQLite file %q: %w", candidate, err) + } + } + return nil +} + +func chmodPrivateFile(path string) error { + if err := os.Chmod(path, 0o600); err != nil { + return err + } + if runtime.GOOS == "windows" { + return nil + } + info, err := os.Stat(path) + if err != nil { + return err + } + if permissions := info.Mode().Perm(); permissions&0o077 != 0 { + return fmt.Errorf("permissions=%#o after chmod 0600", permissions) + } + return nil +} + +func isSQLiteDataFile(name string) bool { + name = strings.ToLower(name) + return strings.HasSuffix(name, ".db") || strings.HasSuffix(name, ".db-wal") || strings.HasSuffix(name, ".db-shm") } func ensureAppUserColumns() error { @@ -724,56 +780,72 @@ func GetCopyIdentityStateWithFolderAlias(accountID int64, folder, legacyFolder s if legacyFolder != "" && legacyFolder != folder { folders = append(folders, legacyFolder) } + aliases := []string{identity.MessageID, identity.LegacyMessageID} + aliases = append(aliases, identity.LegacyMessageIDs...) + if identity.MessageID == "" && identity.BodySHA256 != "" { + aliases = append(aliases, "sha256:"+identity.BodySHA256) + } + uniqueAliases := make([]string, 0, len(aliases)) + seenAliases := map[string]bool{} + for _, alias := range aliases { + alias = strings.TrimSpace(alias) + if alias == "" || seenAliases[alias] { + continue + } + seenAliases[alias] = true + uniqueAliases = append(uniqueAliases, alias) + } // Precise rows always win. For Message-ID-less mail the hash is the whole // identity, irrespective of which legacy key happened to be stored beside it. + // For mail with an ID, historical normalizations are precise aliases only + // when the canonical body hash is identical. var mboxDone, targetDone int for _, candidateFolder := range folders { - var mbox, target int - var err error if identity.MessageID == "" { - err = DB.QueryRow(`SELECT COALESCE(MAX(mbox_done),0), COALESCE(MAX(target_done),0) FROM copied + var mbox, target int + err := DB.QueryRow(`SELECT COALESCE(MAX(mbox_done),0), COALESCE(MAX(target_done),0) FROM copied WHERE account_id=? AND folder=? AND body_sha256=?`, accountID, candidateFolder, identity.BodySHA256).Scan(&mbox, &target) - } else { - err = DB.QueryRow(`SELECT COALESCE(MAX(mbox_done),0), COALESCE(MAX(target_done),0) FROM copied + if err != nil { + return CopyState{}, err + } + mboxDone |= mbox + targetDone |= target + continue + } + for _, alias := range uniqueAliases { + var mbox, target int + err := DB.QueryRow(`SELECT COALESCE(MAX(mbox_done),0), COALESCE(MAX(target_done),0) FROM copied WHERE account_id=? AND folder=? AND message_id=? AND body_sha256=?`, - accountID, candidateFolder, identity.MessageID, identity.BodySHA256).Scan(&mbox, &target) + accountID, candidateFolder, alias, identity.BodySHA256).Scan(&mbox, &target) + if err != nil { + return CopyState{}, err + } + mboxDone |= mbox + targetDone |= target } - if err != nil { - return CopyState{}, err - } - mboxDone |= mbox - targetDone |= target } if mboxDone != 0 || targetDone != 0 { return CopyState{MboxDone: mboxDone != 0, TargetDone: targetDone != 0}, nil } // Once a Message-ID has precise archive identities, a different body with // that same ID is new and must not be hidden by the old wildcard row. - if identity.MessageID != "" && identity.BodySHA256 != "" { + if len(uniqueAliases) != 0 && identity.BodySHA256 != "" { var precise int for _, candidateFolder := range folders { - var count int - if err := DB.QueryRow(`SELECT count(*) FROM copied WHERE account_id=? AND folder=? AND message_id=? AND body_sha256<>''`, - accountID, candidateFolder, identity.MessageID).Scan(&count); err != nil { - return CopyState{}, err + for _, alias := range uniqueAliases { + var count int + if err := DB.QueryRow(`SELECT count(*) FROM copied WHERE account_id=? AND folder=? AND message_id=? AND body_sha256<>''`, + accountID, candidateFolder, alias).Scan(&count); err != nil { + return CopyState{}, err + } + precise += count } - precise += count } if precise > 0 { return CopyState{}, nil } } - aliases := []string{identity.MessageID, identity.LegacyMessageID} - if identity.MessageID == "" && identity.BodySHA256 != "" { - aliases = append(aliases, "sha256:"+identity.BodySHA256) - } - seen := map[string]bool{} - for _, alias := range aliases { - alias = strings.TrimSpace(alias) - if alias == "" || seen[alias] { - continue - } - seen[alias] = true + for _, alias := range uniqueAliases { for _, candidateFolder := range folders { var mbox, target int err := DB.QueryRow(`SELECT COALESCE(MAX(mbox_done),0), COALESCE(MAX(target_done),0) FROM copied @@ -875,21 +947,25 @@ func ReplaceMboxIndex(accountID int64, folder string, entries []MboxIndexEntry, if err := aliasRows.Close(); err != nil { return err } - legacyIDsBySeq := map[int]string{} + type storedIndexIdentity struct { + messageID string + bodySHA256 string + } + storedIdentityBySeq := map[int]storedIndexIdentity{} for _, candidateFolder := range folderAliases { - rows, err := tx.Query(`SELECT seq, message_id FROM mbox_index WHERE account_id=? AND folder=?`, accountID, candidateFolder) + rows, err := tx.Query(`SELECT seq, message_id, body_sha256 FROM mbox_index WHERE account_id=? AND folder=?`, accountID, candidateFolder) if err != nil { return err } for rows.Next() { var seq int - var messageID string - if err := rows.Scan(&seq, &messageID); err != nil { + var identity storedIndexIdentity + if err := rows.Scan(&seq, &identity.messageID, &identity.bodySHA256); err != nil { _ = rows.Close() return err } - if _, exists := legacyIDsBySeq[seq]; !exists || candidateFolder == folder { - legacyIDsBySeq[seq] = messageID + if _, exists := storedIdentityBySeq[seq]; !exists || candidateFolder == folder { + storedIdentityBySeq[seq] = identity } } if err := rows.Err(); err != nil { @@ -907,6 +983,15 @@ func ReplaceMboxIndex(accountID int64, folder string, entries []MboxIndexEntry, entry.AccountID = accountID entry.Folder = folder entry.Seq = i + storedIdentity := storedIdentityBySeq[i] + // The IMAP envelope and a later RFC-822 header parse can represent the + // same malformed Message-ID differently. If the canonical body at the + // same archive position is identical, retain the already committed ID so + // a rebuild is an exact metadata restore. Never carry it across a body + // change, because that could attach an old identity to different content. + if storedIdentity.bodySHA256 != "" && storedIdentity.bodySHA256 == entry.BodySHA256 { + entry.MessageID = storedIdentity.messageID + } if entry.MessageID == "" && entry.BodySHA256 == "" { continue } @@ -921,7 +1006,7 @@ func ReplaceMboxIndex(accountID int64, folder string, entries []MboxIndexEntry, // pipeline the target append happened before the mbox append, so recovered // records are complete in both stages. Marking them prevents a later watch // run from appending duplicates to either destination. - state, found, err := copyStateForReindex(tx, entry.AccountID, folderAliases, entry, legacyIDsBySeq[i]) + state, found, err := copyStateForReindex(tx, entry.AccountID, folderAliases, entry, storedIdentity.messageID) if err != nil { return err } diff --git a/backend/04-imap-source.go b/backend/04-imap-source.go index 46daaad..bc30f44 100644 --- a/backend/04-imap-source.go +++ b/backend/04-imap-source.go @@ -31,10 +31,13 @@ type RawMessage struct { // for byte-different messages. Messages without a Message-ID are identified by // BodySHA256 alone. LegacyMessageID keeps the old raw-IMAP hash addressable // while existing databases are migrated without re-copying mail. +// LegacyMessageIDs additionally retain historical Message-ID normalizations +// from both the IMAP envelope and the raw RFC-822 header. type MessageIdentity struct { - MessageID string - BodySHA256 string - LegacyMessageID string + MessageID string + BodySHA256 string + LegacyMessageID string + LegacyMessageIDs []string } type MessageHeader struct { @@ -489,19 +492,23 @@ func messageID(body []byte) string { func identityForRawMessage(m RawMessage) MessageIdentity { id := normalizeMessageID(m.MessageID) - legacyID := id + legacyID := legacyNormalizeMessageID(m.MessageID) + aliases := []string{legacyID} if strings.HasPrefix(strings.ToLower(id), "sha256:") { id = "" } - if id == "" { - if msg, err := mail.ReadMessage(bytes.NewReader(m.Body)); err == nil { - id = normalizeMessageID(msg.Header.Get("Message-ID")) + if msg, err := mail.ReadMessage(bytes.NewReader(m.Body)); err == nil { + headerID := msg.Header.Get("Message-ID") + aliases = append(aliases, normalizeMessageID(headerID), legacyNormalizeMessageID(headerID)) + if id == "" { + id = normalizeMessageID(headerID) } } return MessageIdentity{ - MessageID: id, - BodySHA256: bodySHA256(m.Body), - LegacyMessageID: legacyID, + MessageID: id, + BodySHA256: bodySHA256(m.Body), + LegacyMessageID: legacyID, + LegacyMessageIDs: aliases, } } @@ -525,6 +532,18 @@ func bodySHA256(raw []byte) string { } func normalizeMessageID(id string) string { + id = strings.TrimSpace(id) + if start := strings.IndexByte(id, '<'); start >= 0 { + if relativeEnd := strings.IndexByte(id[start+1:], '>'); relativeEnd >= 0 { + if candidate := strings.TrimSpace(id[start+1 : start+1+relativeEnd]); candidate != "" { + return candidate + } + } + } + return strings.Trim(id, "<>") +} + +func legacyNormalizeMessageID(id string) string { return strings.Trim(strings.TrimSpace(id), "<>") } diff --git a/backend/04-imap-source_test.go b/backend/04-imap-source_test.go index ef5d5ad..dd27615 100644 --- a/backend/04-imap-source_test.go +++ b/backend/04-imap-source_test.go @@ -25,6 +25,56 @@ func TestMessageIDFallsBackToBodyHash(t *testing.T) { } } +func TestNormalizeMessageIDExtractsBracketedIDBeforeBrokenComment(t *testing.T) { + tests := map[string]string{ + "": "normal@example.com", + "bracketless@example.com": "bracketless@example.com", + " <5486CB7400AF1FB7@mr001msb.fastweb.it> (added by postmaster) ": "5486CB7400AF1FB7@mr001msb.fastweb.it", + "comment trailing": "embedded@example.com", + } + for input, want := range tests { + if got := normalizeMessageID(input); got != want { + t.Errorf("normalizeMessageID(%q)=%q, want %q", input, got, want) + } + } +} + +func TestIdentityKeepsOldMalformedHeaderNormalizationAsAlias(t *testing.T) { + tests := []struct { + header string + clean string + legacy string + }{ + { + header: `<476BDFDB019289BE@mail21.bluewin.ch> (added by postmaster@bluewin.ch)`, + clean: `476BDFDB019289BE@mail21.bluewin.ch`, + legacy: `476BDFDB019289BE@mail21.bluewin.ch> (added by postmaster@bluewin.ch)`, + }, + { + header: `<22578196653698419217.BB8A0E8F48F3F290@dr-gold.de>+D271B1409C3C6028`, + clean: `22578196653698419217.BB8A0E8F48F3F290@dr-gold.de`, + legacy: `22578196653698419217.BB8A0E8F48F3F290@dr-gold.de>+D271B1409C3C6028`, + }, + } + for _, tt := range tests { + body := []byte("Message-ID: " + tt.header + "\r\nSubject: malformed ID\r\n\r\nbody") + identity := identityForRawMessage(RawMessage{MessageID: messageID(body), Body: body}) + if identity.MessageID != tt.clean { + t.Fatalf("MessageID=%q, want %q", identity.MessageID, tt.clean) + } + found := false + for _, alias := range identity.LegacyMessageIDs { + if alias == tt.legacy { + found = true + break + } + } + if !found { + t.Fatalf("legacy alias %q missing from %#v", tt.legacy, identity.LegacyMessageIDs) + } + } +} + func TestBodySHA256IsStableAcrossMboxNormalization(t *testing.T) { raw := []byte("From: a@example.com\r\nSubject: Test\r\n\r\nFirst\r\nFrom escaped\r\n\r\n") record := mboxRecord(RawMessage{Body: raw}) diff --git a/backend/06-mbox.go b/backend/06-mbox.go index 5963fc1..e746e6b 100644 --- a/backend/06-mbox.go +++ b/backend/06-mbox.go @@ -1,6 +1,7 @@ package backend import ( + "bufio" "bytes" "encoding/base64" "fmt" @@ -290,17 +291,55 @@ func reindexAccountArchive(account Account) error { } func reindexPlainMbox(path string) ([]MboxIndexEntry, error) { - b, err := os.ReadFile(path) + f, err := os.Open(path) if err != nil { return nil, err } + defer f.Close() + reader := bufio.NewReaderSize(f, 256*1024) var out []MboxIndexEntry - for _, part := range splitMboxRecordsWithOffsets(b) { - entry := indexEntryFromMboxRecord(part.Message, part.Offset, int64(part.Length), 0, int64(part.Length)) + var record bytes.Buffer + var offset int64 + var messageStart int64 = -1 + appendRecord := func(end int64) { + if messageStart < 0 || end <= messageStart { + return + } + message := unescapeMboxMessage(record.Bytes()) + if len(message) == 0 { + return + } + entry := indexEntryFromMboxRecord(message, messageStart, end-messageStart, 0, end-messageStart) if entry.MessageID != "" || entry.BodySHA256 != "" { out = append(out, entry) } } + for { + lineStart := offset + line, readErr := reader.ReadBytes('\n') + offset += int64(len(line)) + content := line + if len(content) > 0 && content[len(content)-1] == '\n' { + content = content[:len(content)-1] + } + if len(content) > 0 && content[len(content)-1] == '\r' { + content = content[:len(content)-1] + } + if bytes.HasPrefix(content, []byte("From ")) { + appendRecord(lineStart) + record = bytes.Buffer{} + messageStart = offset + } else if messageStart >= 0 && len(line) > 0 { + _, _ = record.Write(line) + } + if readErr != nil { + if readErr != io.EOF { + return nil, readErr + } + break + } + } + appendRecord(offset) return out, nil } diff --git a/backend/06-mbox_test.go b/backend/06-mbox_test.go index 5522bb7..180209f 100644 --- a/backend/06-mbox_test.go +++ b/backend/06-mbox_test.go @@ -3,8 +3,10 @@ package backend import ( "bytes" "database/sql" + "fmt" "os" "path/filepath" + "runtime" "strings" "testing" "time" @@ -12,6 +14,39 @@ import ( "github.com/klauspost/compress/zstd" ) +func TestReindexPlainMboxStreamingMatchesReferenceParser(t *testing.T) { + var mbox bytes.Buffer + for i := 0; i < 257; i++ { + body := strings.Repeat(fmt.Sprintf("line-%03d From inside body\r\n", i), i%31+1) + mbox.Write(mboxRecord(RawMessage{ + MessageID: fmt.Sprintf("stream-%03d@example.com", i), + Body: []byte(fmt.Sprintf( + "Message-ID: \r\nSubject: Stream %03d\r\n\r\n%s", + i, i, body, + )), + })) + } + path := filepath.Join(t.TempDir(), "streaming.mbox") + if err := os.WriteFile(path, mbox.Bytes(), 0o600); err != nil { + t.Fatal(err) + } + + got, err := reindexPlainMbox(path) + if err != nil { + t.Fatal(err) + } + parts := splitMboxRecordsWithOffsets(mbox.Bytes()) + if len(got) != len(parts) { + t.Fatalf("streaming entries=%d, reference parts=%d", len(got), len(parts)) + } + for i, part := range parts { + want := indexEntryFromMboxRecord(part.Message, part.Offset, int64(part.Length), 0, int64(part.Length)) + if got[i] != want { + t.Fatalf("entry %d differs:\n got %#v\n want %#v", i, got[i], want) + } + } +} + func TestZstdMboxRoundTripAndIndexRead(t *testing.T) { oldCfg, oldDB := Cfg, DB root := t.TempDir() @@ -325,6 +360,14 @@ func TestConnectDBConfiguresBusyTimeoutAndCopyStages(t *testing.T) { Cfg, DB = oldCfg, oldDB }) Cfg = Config{DBPath: filepath.Join(root, "mail-graveyard.db"), MboxRoot: filepath.Join(root, "backup")} + backupDB := filepath.Join(root, "PRE-existing-copy.db") + if err := os.WriteFile(backupDB, []byte("diagnostic copy"), 0o644); err != nil { + t.Fatal(err) + } + nonDB := filepath.Join(root, "keep-mode.txt") + if err := os.WriteFile(nonDB, []byte("not a database"), 0o644); err != nil { + t.Fatal(err) + } if err := ConnectDB(true); err != nil { t.Fatal(err) } @@ -335,6 +378,24 @@ func TestConnectDBConfiguresBusyTimeoutAndCopyStages(t *testing.T) { if timeout != 10000 { t.Fatalf("busy_timeout=%d, want 10000", timeout) } + if runtime.GOOS != "windows" { + for _, path := range []string{Cfg.DBPath, Cfg.DBPath + "-wal", Cfg.DBPath + "-shm", backupDB} { + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if permissions := info.Mode().Perm(); permissions != 0o600 { + t.Fatalf("SQLite file %s permissions=%#o, want 0600", path, permissions) + } + } + info, err := os.Stat(nonDB) + if err != nil { + t.Fatal(err) + } + if permissions := info.Mode().Perm(); permissions != 0o644 { + t.Fatalf("non-DB file permissions changed to %#o", permissions) + } + } if err := SaveAccount(Account{ Name: "stage-account", SrcHost: "source.example", SrcPort: 993, SrcSecurity: "tls", SrcUser: "source@example.com", SrcPass: "x", DstHost: "target.example", DstPort: 993, DstSecurity: "tls", DstUser: "target@example.com", DstPass: "x", Active: true, @@ -379,6 +440,106 @@ func TestConnectDBConfiguresBusyTimeoutAndCopyStages(t *testing.T) { } } +func TestReplaceMboxIndexPreservesStoredMessageIDOnlyForSameBody(t *testing.T) { + oldCfg, oldDB := Cfg, DB + root := t.TempDir() + t.Cleanup(func() { + if DB != nil { + _ = DB.Close() + } + Cfg, DB = oldCfg, oldDB + }) + Cfg = Config{DBPath: filepath.Join(root, "mail-graveyard.db"), MboxRoot: filepath.Join(root, "backup")} + if err := ConnectDB(true); err != nil { + t.Fatal(err) + } + if err := SaveAccount(Account{ + Name: "identity-restore", SrcHost: "source.example", SrcPort: 993, SrcSecurity: "tls", SrcUser: "source@example.com", SrcPass: "x", + DstHost: "target.example", DstPort: 993, DstSecurity: "tls", DstUser: "target@example.com", DstPass: "x", Active: true, + }); err != nil { + t.Fatal(err) + } + account, err := GetAccount("identity-restore") + if err != nil { + t.Fatal(err) + } + stableHash := bodySHA256([]byte("same archived body")) + if err := ReplaceMboxIndex(account.ID, "INBOX", []MboxIndexEntry{{ + MessageID: "stored@example.com> (added by postmaster)", BodySHA256: stableHash, FrameLen: 10, InnerLen: 10, + }}, 10); err != nil { + t.Fatal(err) + } + if err := ReplaceMboxIndex(account.ID, "INBOX", []MboxIndexEntry{{ + MessageID: "parsed@example.com", BodySHA256: stableHash, FrameLen: 10, InnerLen: 10, + }}, 10); err != nil { + t.Fatal(err) + } + entry, err := GetMboxIndex(account.ID, "INBOX", 0) + if err != nil { + t.Fatal(err) + } + if entry.MessageID != "stored@example.com> (added by postmaster)" { + t.Fatalf("same body lost stored Message-ID: %q", entry.MessageID) + } + + changedHash := bodySHA256([]byte("different archived body")) + if err := ReplaceMboxIndex(account.ID, "INBOX", []MboxIndexEntry{{ + MessageID: "new-content@example.com", BodySHA256: changedHash, FrameLen: 11, InnerLen: 11, + }}, 11); err != nil { + t.Fatal(err) + } + entry, err = GetMboxIndex(account.ID, "INBOX", 0) + if err != nil { + t.Fatal(err) + } + if entry.MessageID != "new-content@example.com" { + t.Fatalf("different body inherited stale Message-ID: %q", entry.MessageID) + } +} + +func TestPreciseLegacyMessageIDAliasesPreventLiveRecopy(t *testing.T) { + account, _, restore := setupSnapshotTest(t, "none") + defer restore() + tests := []struct { + header string + legacy string + }{ + { + header: `<476BDFDB019289BE@mail21.bluewin.ch> (added by postmaster@bluewin.ch)`, + legacy: `476BDFDB019289BE@mail21.bluewin.ch> (added by postmaster@bluewin.ch)`, + }, + { + header: `<22578196653698419217.BB8A0E8F48F3F290@dr-gold.de>+D271B1409C3C6028`, + legacy: `22578196653698419217.BB8A0E8F48F3F290@dr-gold.de>+D271B1409C3C6028`, + }, + } + for i, tt := range tests { + folder := fmt.Sprintf("INBOX.%d", i) + body := []byte("Message-ID: " + tt.header + "\r\nSubject: malformed ID\r\n\r\noriginal body") + identity := identityForRawMessage(RawMessage{MessageID: messageID(body), Body: body}) + if err := MarkIdentityCopied(account.ID, folder, MessageIdentity{MessageID: tt.legacy, BodySHA256: identity.BodySHA256}); err != nil { + t.Fatal(err) + } + state, err := GetCopyIdentityState(account.ID, folder, identity) + if err != nil { + t.Fatal(err) + } + if !state.MboxDone || !state.TargetDone { + t.Fatalf("fixture %d precise legacy alias was not recognized: %#v", i, state) + } + + changedBody := []byte("Message-ID: " + tt.header + "\r\nSubject: malformed ID\r\n\r\ndifferent body") + changedIdentity := identityForRawMessage(RawMessage{MessageID: messageID(changedBody), Body: changedBody}) + state, err = GetCopyIdentityState(account.ID, folder, changedIdentity) + if err != nil { + t.Fatal(err) + } + if state.MboxDone || state.TargetDone { + t.Fatalf("fixture %d legacy alias hid byte-different mail: %#v", i, state) + } + } +} + func TestIdentitySchemaUpgradePreservesLegacyAliases(t *testing.T) { oldCfg, oldDB := Cfg, DB root := t.TempDir() diff --git a/backend/14-snapshot.go b/backend/14-snapshot.go new file mode 100644 index 0000000..ac8b756 --- /dev/null +++ b/backend/14-snapshot.go @@ -0,0 +1,711 @@ +package backend + +import ( + "crypto/sha256" + "database/sql" + "encoding/binary" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "log" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "time" +) + +const snapshotFormatVersion = 1 + +type SnapshotManifest struct { + FormatVersion int `json:"format_version"` + CreatedAt string `json:"created_at"` + DatabasePath string `json:"database_path"` + DatabaseSHA256 string `json:"database_sha256"` + IndexSHA256 string `json:"index_sha256"` + IndexRecords int64 `json:"index_records"` + PreciseCopied int64 `json:"precise_copied"` + PendingMbox int64 `json:"pending_mbox"` + PendingTarget int64 `json:"pending_target"` + InvalidStageOrder int64 `json:"invalid_stage_order"` + MissingArchiveRecords int64 `json:"missing_archive_records"` + Files []SnapshotFile `json:"files"` +} + +type SnapshotFile struct { + Path string `json:"path"` + Bytes int64 `json:"bytes"` + SHA256 string `json:"sha256"` + AccountID int64 `json:"account_id"` + Account string `json:"account"` + Folder string `json:"folder"` + IndexRecords int64 `json:"index_records"` +} + +type snapshotWatermark struct { + AccountID int64 + Account string + MboxDir string + Folder string + Bytes int64 + IndexRecords int64 + SourcePath string + RelativePath string +} + +// CreateSnapshot creates a self-consistent DB + mbox-prefix snapshot without +// stopping append-only archive writers. The DB snapshot is taken first; every +// copied mbox prefix is then bounded exclusively by offsets from that snapshot. +func CreateSnapshot(target string) error { + return createSnapshot(target, nil) +} + +func createSnapshot(target string, afterDatabaseSnapshot func() error) (retErr error) { + if DB == nil { + return fmt.Errorf("snapshot: database is not connected") + } + target = strings.TrimSpace(target) + if target == "" { + return fmt.Errorf("snapshot: target directory is empty") + } + targetAbs, err := filepath.Abs(target) + if err != nil { + return fmt.Errorf("snapshot target: %w", err) + } + if _, err := os.Stat(targetAbs); err == nil { + return fmt.Errorf("snapshot target already exists: %s", targetAbs) + } else if !os.IsNotExist(err) { + return fmt.Errorf("snapshot target stat: %w", err) + } + if err := os.MkdirAll(filepath.Dir(targetAbs), 0o700); err != nil { + return fmt.Errorf("snapshot target parent: %w", err) + } + partial := fmt.Sprintf("%s.partial-%d-%d", targetAbs, os.Getpid(), time.Now().UnixNano()) + if err := os.Mkdir(partial, 0o700); err != nil { + return fmt.Errorf("snapshot staging directory: %w", err) + } + complete := false + defer func() { + if !complete { + log.Printf("snapshot incomplete; staging data retained at %s", partial) + if retErr != nil { + retErr = fmt.Errorf("%w (incomplete snapshot retained at %s)", retErr, partial) + } + } + }() + + databaseRel := filepath.ToSlash(filepath.Join("data", "mail-graveyard.db")) + databasePath := filepath.Join(partial, filepath.FromSlash(databaseRel)) + if err := os.MkdirAll(filepath.Dir(databasePath), 0o700); err != nil { + return fmt.Errorf("snapshot database directory: %w", err) + } + if _, err := DB.Exec(`VACUUM INTO ?`, databasePath); err != nil { + return fmt.Errorf("snapshot database VACUUM INTO: %w", err) + } + if err := chmodPrivateFile(databasePath); err != nil { + return fmt.Errorf("snapshot database permissions: %w", err) + } + if err := syncFile(databasePath); err != nil { + return fmt.Errorf("snapshot database sync: %w", err) + } + if afterDatabaseSnapshot != nil { + if err := afterDatabaseSnapshot(); err != nil { + return fmt.Errorf("snapshot test hook: %w", err) + } + } + + snapshotDB, err := openSnapshotDatabase(databasePath) + if err != nil { + return err + } + watermarks, err := collectSnapshotWatermarks(snapshotDB, Cfg.MboxRoot) + if err != nil { + _ = snapshotDB.Close() + return err + } + stats, err := snapshotDatabaseStats(snapshotDB) + if err == nil { + stats.IndexSHA256, err = indexDigest(snapshotDB) + } + if closeErr := snapshotDB.Close(); err == nil && closeErr != nil { + err = closeErr + } + if err != nil { + return err + } + if stats.MissingArchiveRecords != 0 { + return fmt.Errorf("snapshot refused: %d mbox_done identities have no archive index", stats.MissingArchiveRecords) + } + if stats.InvalidStageOrder != 0 { + return fmt.Errorf("snapshot refused: %d precise identities have target_done=1 before mbox_done", stats.InvalidStageOrder) + } + var watermarkRecords int64 + for _, watermark := range watermarks { + watermarkRecords += watermark.IndexRecords + } + if watermarkRecords != stats.IndexRecords { + return fmt.Errorf("snapshot refused: indexed records=%d but mapped mbox records=%d", stats.IndexRecords, watermarkRecords) + } + + manifest := SnapshotManifest{ + FormatVersion: snapshotFormatVersion, + CreatedAt: time.Now().UTC().Format(time.RFC3339Nano), + DatabasePath: databaseRel, + IndexSHA256: stats.IndexSHA256, + IndexRecords: stats.IndexRecords, + PreciseCopied: stats.PreciseCopied, + PendingMbox: stats.PendingMbox, + PendingTarget: stats.PendingTarget, + InvalidStageOrder: stats.InvalidStageOrder, + MissingArchiveRecords: stats.MissingArchiveRecords, + Files: make([]SnapshotFile, 0, len(watermarks)), + } + manifest.DatabaseSHA256, err = hashFile(databasePath) + if err != nil { + return fmt.Errorf("snapshot database hash: %w", err) + } + for _, watermark := range watermarks { + destinationRel := filepath.ToSlash(filepath.Join("backup", watermark.RelativePath)) + destination := filepath.Join(partial, filepath.FromSlash(destinationRel)) + hash, err := copyFilePrefix(watermark.SourcePath, destination, watermark.Bytes) + if err != nil { + return fmt.Errorf("snapshot mbox %s: %w", destinationRel, err) + } + manifest.Files = append(manifest.Files, SnapshotFile{ + Path: destinationRel, + Bytes: watermark.Bytes, + SHA256: hash, + AccountID: watermark.AccountID, + Account: watermark.Account, + Folder: watermark.Folder, + IndexRecords: watermark.IndexRecords, + }) + } + sortSnapshotFiles(manifest.Files) + manifestPath := filepath.Join(partial, "snapshot-manifest.json") + manifestJSON, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return fmt.Errorf("snapshot manifest encode: %w", err) + } + manifestJSON = append(manifestJSON, '\n') + if err := os.WriteFile(manifestPath, manifestJSON, 0o600); err != nil { + return fmt.Errorf("snapshot manifest write: %w", err) + } + if err := syncFile(manifestPath); err != nil { + return fmt.Errorf("snapshot manifest sync: %w", err) + } + if err := os.Rename(partial, targetAbs); err != nil { + return fmt.Errorf("snapshot finalize: %w", err) + } + complete = true + log.Printf("snapshot complete target=%s files=%d bytes=%d index_records=%d precise_copied=%d pending_mbox=%d pending_target=%d", + targetAbs, len(manifest.Files), snapshotBytes(manifest.Files), manifest.IndexRecords, manifest.PreciseCopied, manifest.PendingMbox, manifest.PendingTarget) + return nil +} + +type snapshotStats struct { + IndexRecords int64 + PreciseCopied int64 + PendingMbox int64 + PendingTarget int64 + MissingArchiveRecords int64 + InvalidStageOrder int64 + IndexSHA256 string +} + +func snapshotDatabaseStats(db *sql.DB) (snapshotStats, error) { + var stats snapshotStats + queries := []struct { + query string + dest *int64 + }{ + {`SELECT count(*) FROM mbox_index`, &stats.IndexRecords}, + {`SELECT count(*) FROM copied WHERE body_sha256<>''`, &stats.PreciseCopied}, + {`SELECT count(*) FROM copied WHERE body_sha256<>'' AND mbox_done=0`, &stats.PendingMbox}, + {`SELECT count(*) FROM copied WHERE body_sha256<>'' AND target_done=0`, &stats.PendingTarget}, + {`SELECT count(*) FROM copied WHERE body_sha256<>'' AND target_done=1 AND mbox_done=0`, &stats.InvalidStageOrder}, + {`SELECT count(*) FROM copied c WHERE c.body_sha256<>'' AND c.mbox_done=1 AND NOT EXISTS ( + SELECT 1 FROM mbox_index i + WHERE i.account_id=c.account_id AND i.folder=c.folder + AND i.message_id=c.message_id AND i.body_sha256=c.body_sha256 + )`, &stats.MissingArchiveRecords}, + } + for _, item := range queries { + if err := db.QueryRow(item.query).Scan(item.dest); err != nil { + return snapshotStats{}, fmt.Errorf("snapshot database statistics: %w", err) + } + } + return stats, nil +} + +func indexDigest(db *sql.DB) (string, error) { + rows, err := db.Query(`SELECT account_id, folder, seq, message_id, body_sha256 + FROM mbox_index ORDER BY account_id, folder, seq`) + if err != nil { + return "", fmt.Errorf("snapshot index digest query: %w", err) + } + defer rows.Close() + hash := sha256.New() + for rows.Next() { + var accountID, seq int64 + var folder, messageID, bodyHash string + if err := rows.Scan(&accountID, &folder, &seq, &messageID, &bodyHash); err != nil { + return "", fmt.Errorf("snapshot index digest row: %w", err) + } + for _, value := range []int64{accountID, seq} { + if err := binary.Write(hash, binary.LittleEndian, value); err != nil { + return "", err + } + } + for _, value := range []string{folder, messageID, bodyHash} { + if err := binary.Write(hash, binary.LittleEndian, uint64(len(value))); err != nil { + return "", err + } + if _, err := io.WriteString(hash, value); err != nil { + return "", err + } + } + } + if err := rows.Err(); err != nil { + return "", fmt.Errorf("snapshot index digest: %w", err) + } + return hex.EncodeToString(hash.Sum(nil)), nil +} + +func collectSnapshotWatermarks(db *sql.DB, mboxRoot string) ([]snapshotWatermark, error) { + rootAbs, err := filepath.Abs(mboxRoot) + if err != nil { + return nil, fmt.Errorf("snapshot mbox root: %w", err) + } + rows, err := db.Query(`SELECT i.account_id, a.name, a.mbox_dir, i.folder, + MAX(i.file_offset+i.frame_len), count(*) + FROM mbox_index i JOIN accounts a ON a.id=i.account_id + GROUP BY i.account_id, a.name, a.mbox_dir, i.folder + ORDER BY lower(a.name), a.name, i.folder`) + if err != nil { + return nil, fmt.Errorf("snapshot watermarks: %w", err) + } + defer rows.Close() + var out []snapshotWatermark + seenPaths := map[string]bool{} + archivePaths := map[string]map[string]string{} + for rows.Next() { + var watermark snapshotWatermark + if err := rows.Scan(&watermark.AccountID, &watermark.Account, &watermark.MboxDir, &watermark.Folder, &watermark.Bytes, &watermark.IndexRecords); err != nil { + return nil, fmt.Errorf("snapshot watermark row: %w", err) + } + watermark.MboxDir = strings.TrimSpace(watermark.MboxDir) + if watermark.MboxDir == "" { + return nil, fmt.Errorf("snapshot account %s has no mbox_dir", watermark.Account) + } + paths, ok := archivePaths[watermark.MboxDir] + if !ok { + paths = map[string]string{} + for _, path := range archiveMboxPaths(filepath.Join(rootAbs, watermark.MboxDir)) { + paths[folderNameFromMboxPath(path)] = path + } + archivePaths[watermark.MboxDir] = paths + } + watermark.SourcePath = paths[watermark.Folder] + if watermark.SourcePath == "" { + return nil, fmt.Errorf("snapshot indexed mbox missing for account=%s folder=%q", watermark.Account, watermark.Folder) + } + pathAbs, err := filepath.Abs(watermark.SourcePath) + if err != nil { + return nil, fmt.Errorf("snapshot mbox path: %w", err) + } + relative, err := filepath.Rel(rootAbs, pathAbs) + if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(os.PathSeparator)) || filepath.IsAbs(relative) { + return nil, fmt.Errorf("snapshot mbox escapes configured root: %s", pathAbs) + } + watermark.SourcePath = pathAbs + watermark.RelativePath = relative + pathKey := snapshotPathKey(relative) + if seenPaths[pathKey] { + return nil, fmt.Errorf("snapshot multiple index groups resolve to the same mbox: %s", relative) + } + seenPaths[pathKey] = true + info, err := os.Stat(pathAbs) + if err != nil { + return nil, fmt.Errorf("snapshot mbox stat %s: %w", pathAbs, err) + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("snapshot mbox is not a regular file: %s", pathAbs) + } + if watermark.Bytes <= 0 || info.Size() < watermark.Bytes { + return nil, fmt.Errorf("snapshot invalid watermark account=%s folder=%q boundary=%d size=%d", + watermark.Account, watermark.Folder, watermark.Bytes, info.Size()) + } + out = append(out, watermark) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("snapshot watermarks: %w", err) + } + return out, nil +} + +func copyFilePrefix(source, destination string, length int64) (string, error) { + if length < 0 { + return "", fmt.Errorf("negative prefix length %d", length) + } + if err := os.MkdirAll(filepath.Dir(destination), 0o700); err != nil { + return "", err + } + src, err := os.Open(source) + if err != nil { + return "", err + } + defer src.Close() + dst, err := os.OpenFile(destination, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return "", err + } + hash := sha256.New() + written, copyErr := io.CopyN(io.MultiWriter(dst, hash), src, length) + if copyErr == nil && written != length { + copyErr = io.ErrShortWrite + } + if copyErr == nil { + copyErr = dst.Sync() + } + closeErr := dst.Close() + if copyErr != nil { + return "", copyErr + } + if closeErr != nil { + return "", closeErr + } + return hex.EncodeToString(hash.Sum(nil)), nil +} + +func openSnapshotDatabase(path string) (*sql.DB, error) { + db, err := sql.Open("sqlite", path) + if err != nil { + return nil, fmt.Errorf("open snapshot database: %w", err) + } + db.SetMaxOpenConns(1) + if _, err := db.Exec(`PRAGMA query_only=ON; PRAGMA foreign_keys=ON;`); err != nil { + _ = db.Close() + return nil, fmt.Errorf("open snapshot database read-only: %w", err) + } + return db, nil +} + +// VerifySnapshot validates hashes and DB/file boundaries, then copies only the +// small DB into a temporary restore workspace and performs a full reindex there. +// The snapshot mbox files and any live database remain untouched. +func VerifySnapshot(root string) error { + root = strings.TrimSpace(root) + if root == "" { + return fmt.Errorf("verify snapshot: directory is empty") + } + rootAbs, err := filepath.Abs(root) + if err != nil { + return fmt.Errorf("verify snapshot path: %w", err) + } + manifestBytes, err := os.ReadFile(filepath.Join(rootAbs, "snapshot-manifest.json")) + if err != nil { + return fmt.Errorf("verify snapshot manifest: %w", err) + } + var manifest SnapshotManifest + if err := json.Unmarshal(manifestBytes, &manifest); err != nil { + return fmt.Errorf("verify snapshot manifest decode: %w", err) + } + if manifest.FormatVersion != snapshotFormatVersion { + return fmt.Errorf("verify snapshot format=%d, supported=%d", manifest.FormatVersion, snapshotFormatVersion) + } + databasePath, err := pathInsideSnapshot(rootAbs, manifest.DatabasePath) + if err != nil { + return err + } + if runtime.GOOS != "windows" { + databaseInfo, err := os.Stat(databasePath) + if err != nil { + return fmt.Errorf("verify snapshot database permissions: %w", err) + } + if permissions := databaseInfo.Mode().Perm(); permissions&0o077 != 0 { + return fmt.Errorf("verify snapshot database permissions=%#o, want no group/other access", permissions) + } + } + databaseHash, err := hashFile(databasePath) + if err != nil { + return fmt.Errorf("verify snapshot database hash: %w", err) + } + if databaseHash != manifest.DatabaseSHA256 { + return fmt.Errorf("verify snapshot database hash mismatch: got %s want %s", databaseHash, manifest.DatabaseSHA256) + } + for _, file := range manifest.Files { + path, err := pathInsideSnapshot(rootAbs, file.Path) + if err != nil { + return err + } + info, err := os.Stat(path) + if err != nil { + return fmt.Errorf("verify snapshot file %s: %w", file.Path, err) + } + if info.Size() != file.Bytes { + return fmt.Errorf("verify snapshot file size %s: got %d want %d", file.Path, info.Size(), file.Bytes) + } + hash, err := hashFile(path) + if err != nil { + return fmt.Errorf("verify snapshot file hash %s: %w", file.Path, err) + } + if hash != file.SHA256 { + return fmt.Errorf("verify snapshot file hash mismatch %s: got %s want %s", file.Path, hash, file.SHA256) + } + } + + snapshotDB, err := openSnapshotDatabase(databasePath) + if err != nil { + return err + } + var integrity string + if err := snapshotDB.QueryRow(`PRAGMA integrity_check`).Scan(&integrity); err != nil { + _ = snapshotDB.Close() + return fmt.Errorf("verify snapshot integrity: %w", err) + } + if integrity != "ok" { + _ = snapshotDB.Close() + return fmt.Errorf("verify snapshot integrity: %s", integrity) + } + stats, err := snapshotDatabaseStats(snapshotDB) + if err == nil { + stats.IndexSHA256, err = indexDigest(snapshotDB) + } + if err != nil { + _ = snapshotDB.Close() + return err + } + watermarks, err := collectSnapshotWatermarks(snapshotDB, filepath.Join(rootAbs, "backup")) + if closeErr := snapshotDB.Close(); err == nil && closeErr != nil { + err = closeErr + } + if err != nil { + return err + } + if stats.IndexRecords != manifest.IndexRecords || stats.PreciseCopied != manifest.PreciseCopied || + stats.PendingMbox != manifest.PendingMbox || stats.PendingTarget != manifest.PendingTarget || + stats.InvalidStageOrder != manifest.InvalidStageOrder || stats.MissingArchiveRecords != manifest.MissingArchiveRecords || + stats.IndexSHA256 != manifest.IndexSHA256 { + return fmt.Errorf("verify snapshot manifest/database statistics differ") + } + if stats.MissingArchiveRecords != 0 { + return fmt.Errorf("verify snapshot has %d mbox_done identities without archive index", stats.MissingArchiveRecords) + } + if stats.InvalidStageOrder != 0 { + return fmt.Errorf("verify snapshot has %d precise identities with invalid stage order", stats.InvalidStageOrder) + } + if err := compareManifestWatermarks(manifest.Files, watermarks); err != nil { + return err + } + if err := verifySnapshotReindex(rootAbs, databasePath, manifest.IndexRecords, manifest.IndexSHA256); err != nil { + return err + } + log.Printf("snapshot verified root=%s files=%d bytes=%d index_records=%d precise_copied=%d pending_mbox=%d pending_target=%d", + rootAbs, len(manifest.Files), snapshotBytes(manifest.Files), manifest.IndexRecords, manifest.PreciseCopied, manifest.PendingMbox, manifest.PendingTarget) + return nil +} + +func compareManifestWatermarks(files []SnapshotFile, watermarks []snapshotWatermark) error { + manifestByPath := make(map[string]SnapshotFile, len(files)) + for _, file := range files { + key := snapshotPathKey(filepath.FromSlash(file.Path)) + if _, exists := manifestByPath[key]; exists { + return fmt.Errorf("verify snapshot duplicate manifest path: %s", file.Path) + } + manifestByPath[key] = file + } + if len(manifestByPath) != len(watermarks) { + return fmt.Errorf("verify snapshot file/watermark count differs: manifest=%d database=%d", len(manifestByPath), len(watermarks)) + } + for _, watermark := range watermarks { + path := filepath.Join("backup", watermark.RelativePath) + key := snapshotPathKey(path) + file, ok := manifestByPath[key] + if !ok { + return fmt.Errorf("verify snapshot manifest missing DB watermark: %s", filepath.ToSlash(path)) + } + if file.Bytes != watermark.Bytes || file.IndexRecords != watermark.IndexRecords || file.AccountID != watermark.AccountID || file.Folder != watermark.Folder { + return fmt.Errorf("verify snapshot watermark mismatch: %s", file.Path) + } + } + return nil +} + +func verifySnapshotReindex(root, databasePath string, expectedIndexRecords int64, expectedIndexSHA256 string) error { + temporaryRoot, err := os.MkdirTemp("", "mail-graveyard-snapshot-verify-") + if err != nil { + return fmt.Errorf("verify snapshot temporary restore: %w", err) + } + defer os.RemoveAll(temporaryRoot) + temporaryDB := filepath.Join(temporaryRoot, "mail-graveyard.db") + databaseInfo, err := os.Stat(databasePath) + if err != nil { + return fmt.Errorf("verify snapshot temporary database stat: %w", err) + } + if _, err := copyFilePrefix(databasePath, temporaryDB, databaseInfo.Size()); err != nil { + return fmt.Errorf("verify snapshot temporary database copy: %w", err) + } + oldCfg, oldDB := Cfg, DB + Cfg.DBPath = temporaryDB + Cfg.MboxRoot = filepath.Join(root, "backup") + DB = nil + defer func() { + if DB != nil { + _ = DB.Close() + } + Cfg, DB = oldCfg, oldDB + }() + if err := ConnectDB(false); err != nil { + return fmt.Errorf("verify snapshot restore database: %w", err) + } + if _, err := DB.Exec(`CREATE TABLE snapshot_verify_expected_index AS + SELECT account_id, folder, seq, message_id, body_sha256 FROM mbox_index`); err != nil { + return fmt.Errorf("verify snapshot preserve expected index: %w", err) + } + if err := ReindexArchives("all"); err != nil { + return fmt.Errorf("verify snapshot restore reindex: %w", err) + } + var reindexed int64 + if err := DB.QueryRow(`SELECT count(*) FROM mbox_index`).Scan(&reindexed); err != nil { + return fmt.Errorf("verify snapshot restore index count: %w", err) + } + if reindexed != expectedIndexRecords { + return fmt.Errorf("verify snapshot restore index count=%d, want %d", reindexed, expectedIndexRecords) + } + reindexedDigest, err := indexDigest(DB) + if err != nil { + return err + } + if reindexedDigest != expectedIndexSHA256 { + difference, differenceErr := snapshotIndexDifference(DB) + if differenceErr != nil { + return fmt.Errorf("verify snapshot restore index digest=%s, want %s (difference query: %v)", reindexedDigest, expectedIndexSHA256, differenceErr) + } + return fmt.Errorf("verify snapshot restore index digest=%s, want %s (%s)", reindexedDigest, expectedIndexSHA256, difference) + } + stats, err := snapshotDatabaseStats(DB) + if err != nil { + return err + } + if stats.MissingArchiveRecords != 0 { + return fmt.Errorf("verify snapshot restore has %d copied identities without archive index", stats.MissingArchiveRecords) + } + if stats.InvalidStageOrder != 0 { + return fmt.Errorf("verify snapshot restore has %d precise identities with invalid stage order", stats.InvalidStageOrder) + } + return nil +} + +func snapshotIndexDifference(db *sql.DB) (string, error) { + type differenceRow struct { + accountID int64 + folder string + seq int64 + messageID string + bodyHash string + } + queries := []struct { + label string + left string + right string + }{ + {"missing after reindex", "snapshot_verify_expected_index", "mbox_index"}, + {"unexpected after reindex", "mbox_index", "snapshot_verify_expected_index"}, + } + parts := make([]string, 0, len(queries)) + for _, query := range queries { + statement := fmt.Sprintf(`SELECT account_id, folder, seq, message_id, body_sha256 FROM %s + EXCEPT SELECT account_id, folder, seq, message_id, body_sha256 FROM %s`, query.left, query.right) + var count int64 + if err := db.QueryRow(`SELECT count(*) FROM (` + statement + `)`).Scan(&count); err != nil { + return "", err + } + rows, err := db.Query(statement + ` LIMIT 3`) + if err != nil { + return "", err + } + var samples []string + for rows.Next() { + var row differenceRow + if err := rows.Scan(&row.accountID, &row.folder, &row.seq, &row.messageID, &row.bodyHash); err != nil { + _ = rows.Close() + return "", err + } + hash := row.bodyHash + if len(hash) > 16 { + hash = hash[:16] + } + samples = append(samples, fmt.Sprintf("account=%d folder=%q seq=%d id=%q hash=%s", row.accountID, row.folder, row.seq, row.messageID, hash)) + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return "", err + } + if err := rows.Close(); err != nil { + return "", err + } + parts = append(parts, fmt.Sprintf("%s=%d samples=[%s]", query.label, count, strings.Join(samples, "; "))) + } + return strings.Join(parts, ", "), nil +} + +func pathInsideSnapshot(root, relative string) (string, error) { + if relative == "" || filepath.IsAbs(relative) { + return "", fmt.Errorf("verify snapshot invalid relative path: %q", relative) + } + path := filepath.Join(root, filepath.FromSlash(relative)) + pathAbs, err := filepath.Abs(path) + if err != nil { + return "", err + } + rel, err := filepath.Rel(root, pathAbs) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { + return "", fmt.Errorf("verify snapshot path escapes root: %q", relative) + } + return pathAbs, nil +} + +func hashFile(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + hash := sha256.New() + if _, err := io.Copy(hash, f); err != nil { + return "", err + } + return hex.EncodeToString(hash.Sum(nil)), nil +} + +func syncFile(path string) error { + f, err := os.OpenFile(path, os.O_RDWR, 0) + if err != nil { + return err + } + defer f.Close() + return f.Sync() +} + +func snapshotBytes(files []SnapshotFile) int64 { + var total int64 + for _, file := range files { + total += file.Bytes + } + return total +} + +func snapshotPathKey(path string) string { + path = filepath.Clean(path) + if runtime.GOOS == "windows" { + return strings.ToLower(path) + } + return path +} + +func sortSnapshotFiles(files []SnapshotFile) { + sort.Slice(files, func(i, j int) bool { return files[i].Path < files[j].Path }) +} diff --git a/backend/14-snapshot_test.go b/backend/14-snapshot_test.go new file mode 100644 index 0000000..0e03af9 --- /dev/null +++ b/backend/14-snapshot_test.go @@ -0,0 +1,209 @@ +package backend + +import ( + "encoding/json" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +func TestSnapshotUsesDatabaseWatermarkWhileArchiveKeepsGrowing(t *testing.T) { + for _, compression := range []string{"none", "zstd"} { + t.Run(compression, func(t *testing.T) { + account, writer, restore := setupSnapshotTest(t, compression) + defer restore() + + first := RawMessage{ + MessageID: "first@example.com", + Body: []byte("Message-ID: \r\nSubject: First\r\n\r\nFirst body\r\nFrom quoted\r\n"), + InternalDate: time.Unix(1_700_000_000, 0), + } + firstInfo := appendSnapshotTestMessage(t, account, writer, "INBOX", first) + snapshotRoot := filepath.Join(t.TempDir(), "snapshot") + var secondInfo MboxAppendInfo + err := createSnapshot(snapshotRoot, func() error { + second := RawMessage{ + MessageID: "second@example.com", + Body: []byte("Message-ID: \r\nSubject: Second\r\n\r\nSecond body"), + InternalDate: time.Unix(1_700_000_100, 0), + } + secondInfo = appendSnapshotTestMessage(t, account, writer, "INBOX", second) + return nil + }) + if err != nil { + t.Fatal(err) + } + + manifest := readSnapshotTestManifest(t, snapshotRoot) + if runtime.GOOS != "windows" { + databaseInfo, err := os.Stat(filepath.Join(snapshotRoot, filepath.FromSlash(manifest.DatabasePath))) + if err != nil { + t.Fatal(err) + } + if permissions := databaseInfo.Mode().Perm(); permissions != 0o600 { + t.Fatalf("snapshot database permissions=%#o, want 0600", permissions) + } + } + if manifest.IndexRecords != 1 || manifest.PreciseCopied != 1 || len(manifest.Files) != 1 { + t.Fatalf("snapshot crossed DB boundary: %#v", manifest) + } + if manifest.Files[0].Bytes != firstInfo.FileOffset+firstInfo.FrameLen { + t.Fatalf("snapshot bytes=%d, want first committed boundary=%d", manifest.Files[0].Bytes, firstInfo.FileOffset+firstInfo.FrameLen) + } + sourceInfo, err := os.Stat(secondInfo.Path) + if err != nil { + t.Fatal(err) + } + if sourceInfo.Size() <= manifest.Files[0].Bytes { + t.Fatalf("source did not grow beyond snapshot boundary: source=%d snapshot=%d", sourceInfo.Size(), manifest.Files[0].Bytes) + } + snapshotMbox := filepath.Join(snapshotRoot, filepath.FromSlash(manifest.Files[0].Path)) + messages, err := readMboxMessages(snapshotMbox) + if err != nil { + t.Fatal(err) + } + if len(messages) != 1 || !strings.Contains(string(messages[0]), "First body") { + t.Fatalf("snapshot messages=%d, expected only first message", len(messages)) + } + if err := VerifySnapshot(snapshotRoot); err != nil { + t.Fatalf("verify valid snapshot: %v", err) + } + var liveIndex int + if err := DB.QueryRow(`SELECT count(*) FROM mbox_index WHERE account_id=?`, account.ID).Scan(&liveIndex); err != nil { + t.Fatal(err) + } + if liveIndex != 2 { + t.Fatalf("snapshot verification altered live database: index=%d", liveIndex) + } + + f, err := os.OpenFile(snapshotMbox, os.O_APPEND|os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + if _, err := f.Write([]byte("tamper")); err != nil { + _ = f.Close() + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + if err := VerifySnapshot(snapshotRoot); err == nil { + t.Fatal("verification accepted a modified mbox") + } + }) + } +} + +func TestSnapshotRefusesKnownMissingArchiveIdentity(t *testing.T) { + account, _, restore := setupSnapshotTest(t, "none") + defer restore() + identity := MessageIdentity{MessageID: "missing@example.com", BodySHA256: bodySHA256([]byte("missing"))} + if err := MarkIdentityCopied(account.ID, "INBOX", identity); err != nil { + t.Fatal(err) + } + target := filepath.Join(t.TempDir(), "refused") + err := CreateSnapshot(target) + if err == nil || !strings.Contains(err.Error(), "no archive index") { + t.Fatalf("snapshot error=%v, want missing archive refusal", err) + } + if _, statErr := os.Stat(target); !os.IsNotExist(statErr) { + t.Fatalf("refused snapshot unexpectedly finalized: %v", statErr) + } +} + +func TestSnapshotDoesNotOverwriteExistingTarget(t *testing.T) { + _, _, restore := setupSnapshotTest(t, "none") + defer restore() + target := filepath.Join(t.TempDir(), "existing") + if err := os.Mkdir(target, 0o700); err != nil { + t.Fatal(err) + } + sentinel := filepath.Join(target, "keep") + if err := os.WriteFile(sentinel, []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + if err := CreateSnapshot(target); err == nil { + t.Fatal("snapshot overwrote an existing target") + } + if got, err := os.ReadFile(sentinel); err != nil || string(got) != "keep" { + t.Fatalf("existing target changed: %q %v", got, err) + } +} + +func setupSnapshotTest(t *testing.T, compression string) (Account, *MboxWriter, func()) { + t.Helper() + oldCfg, oldDB := Cfg, DB + root := t.TempDir() + Cfg = Config{ + DBPath: filepath.Join(root, "mail-graveyard.db"), + MboxRoot: filepath.Join(root, "backup"), + MboxCompression: compression, + } + DB = nil + if err := ConnectDB(true); err != nil { + t.Fatal(err) + } + restore := func() { + if DB != nil { + _ = DB.Close() + } + Cfg, DB = oldCfg, oldDB + } + if err := SaveAccount(Account{ + Name: "snapshot-account", SrcHost: "source.example", SrcPort: 993, SrcSecurity: "tls", SrcUser: "source@example.com", SrcPass: "x", + DstHost: "target.example", DstPort: 993, DstSecurity: "tls", DstUser: "target@example.com", DstPass: "x", + MboxDir: "snapshot@example.com", Active: true, + }); err != nil { + restore() + t.Fatal(err) + } + account, err := GetAccount("snapshot-account") + if err != nil { + restore() + t.Fatal(err) + } + writer, err := NewMboxWriter(filepath.Join(Cfg.MboxRoot, account.MboxDir)) + if err != nil { + restore() + t.Fatal(err) + } + return account, writer, restore +} + +func appendSnapshotTestMessage(t *testing.T, account Account, writer *MboxWriter, folder string, message RawMessage) MboxAppendInfo { + t.Helper() + identity := identityForRawMessage(message) + info, err := writer.Append(folder, message) + if err != nil { + t.Fatal(err) + } + if err := SaveMboxIndex(MboxIndexEntry{ + AccountID: account.ID, Folder: safeMboxName(folder), MessageID: identity.MessageID, BodySHA256: identity.BodySHA256, + FileOffset: info.FileOffset, FrameLen: info.FrameLen, InnerOffset: info.InnerOffset, InnerLen: info.InnerLen, + }); err != nil { + t.Fatal(err) + } + if err := UpdateMboxIndexState(account.ID, safeMboxName(folder), info.FileOffset+info.FrameLen); err != nil { + t.Fatal(err) + } + if err := MarkIdentityCopied(account.ID, safeMboxName(folder), identity); err != nil { + t.Fatal(err) + } + return info +} + +func readSnapshotTestManifest(t *testing.T, root string) SnapshotManifest { + t.Helper() + b, err := os.ReadFile(filepath.Join(root, "snapshot-manifest.json")) + if err != nil { + t.Fatal(err) + } + var manifest SnapshotManifest + if err := json.Unmarshal(b, &manifest); err != nil { + t.Fatal(err) + } + return manifest +} diff --git a/main.go b/main.go index 6711dc4..8001ec4 100644 --- a/main.go +++ b/main.go @@ -25,16 +25,32 @@ func main() { checkAccount := flag.String("check-account", "", "Konto-Logins pruefen und INBOX zaehlen, ohne zu kopieren") reindex := flag.String("reindex", "", "Archiv-mbox-Index fuer Konto/Archiv oder 'all' neu aufbauen") dedupTarget := flag.String("dedup-target", "", "Ziel-Postfach nach Message-ID-Dubletten scannen (Dry-Run ohne --apply)") + snapshot := flag.String("snapshot", "", "konsistenten DB- und mbox-Praefix-Snapshot in ein neues Zielverzeichnis schreiben") + verifySnapshot := flag.String("verify-snapshot", "", "Snapshot isoliert pruefen und dessen DB-Kopie vollstaendig reindexieren") apply := flag.Bool("apply", false, "Scharfe Ausfuehrung fuer riskante Wartungsbefehle wie --dedup-target") initDB := flag.Bool("init-db", false, "SQLite-DB neu anlegen, falls db_path noch nicht existiert") flag.Parse() + if *snapshot != "" && (*verifySnapshot != "" || *runOnce != "" || *seedAccount != "" || *checkAccount != "" || *reindex != "" || *dedupTarget != "" || *watch) { + log.Fatal("-snapshot kann nicht mit einem anderen Betriebsmodus kombiniert werden") + } + if *verifySnapshot != "" && (*runOnce != "" || *seedAccount != "" || *checkAccount != "" || *reindex != "" || *dedupTarget != "" || *watch) { + log.Fatal("-verify-snapshot kann nicht mit einem anderen Betriebsmodus kombiniert werden") + } if err := backend.LoadConfig("config.json"); err != nil { log.Fatalf("config: %v", err) } + if *verifySnapshot != "" { + must(backend.VerifySnapshot(*verifySnapshot)) + return + } if err := backend.ConnectDB(*initDB); err != nil { log.Fatalf("database: %v", err) } + if *snapshot != "" { + must(backend.CreateSnapshot(*snapshot)) + return + } if err := backend.InitAuth(); err != nil { log.Fatalf("auth: %v", err) }