Add verified snapshots and harden archive identity

This commit is contained in:
DonVoo 2026-07-17 00:37:02 +02:00
parent c6de214ac8
commit 68b9cee880
9 changed files with 1372 additions and 52 deletions

View file

@ -47,8 +47,10 @@ Ohne Web, z. B. als Cronjob oder Dauerlauf im Umzugsfenster:
./mail-graveyard --run dr-gold --watch # ein Konto, Delta-Schleife bis Stop
```
Dank Message-ID-Cache kopiert jeder Lauf nur Neues — beliebig oft wiederholbar
ohne Dubletten. Für den Dauerlauf reicht `nohup`/systemd/Aufgabenplanung.
Die kanonische Inhaltsidentitaet verhindert dabei doppelte **Schreibvorgaenge**
in Archiv und Ziel. Der aktuelle Volllauf laedt trotzdem jede Quellmail erneut,
bevor er deren Body-Hash pruefen kann. Grosse Konten deshalb bis zum
UID-Delta-Watcher nicht in kurzen Intervallen als Dauerlauf starten.
## 5. Wartung
@ -56,18 +58,46 @@ ohne Dubletten. Für den Dauerlauf reicht `nohup`/systemd/Aufgabenplanung.
./mail-graveyard --reindex all
./mail-graveyard --dedup-target konto-name # Dry-Run
./mail-graveyard --dedup-target konto-name --apply # loescht Dubletten im Ziel
./mail-graveyard --snapshot /sicherungen/stand-2026-07-16
./mail-graveyard --verify-snapshot /sicherungen/stand-2026-07-16
```
`--reindex` baut den Archiv-Index fuer vorhandene mbox/mbox.zst-Dateien neu auf.
`--dedup-target` ist bewusst ein Dry-Run, bis `--apply` gesetzt wird; Mails ohne
Message-ID werden nie geloescht.
`--snapshot` erzeugt zuerst per SQLite `VACUUM INTO` einen konsistenten
DB-Stand. Anschliessend kopiert es aus jeder append-only mbox exakt das durch
`MAX(file_offset+frame_len)` **dieses DB-Standes** belegte Praefix. Das Ziel
muss neu sein und enthaelt `data/mail-graveyard.db`, `backup/...` sowie ein
Manifest mit Grenzen und SHA-256-Pruefsummen. Ein Fehler hinterlaesst ein klar
benanntes `.partial-*`-Verzeichnis und wird nie als fertiger Stand umbenannt.
Die Datenbank wird wegen der enthaltenen Klartext-Zugangsdaten mit `0600`
geschrieben; die Verifikation weist unter Linux weiter gefasste Rechte zurueck.
`--verify-snapshot` veraendert weder Produktion noch Snapshot: Es prueft
Manifest, Dateigroessen, SHA-256, SQLite-Integritaet, Stufenreihenfolge und den
Identitaetsdigest. Danach kopiert es nur die Snapshot-DB in ein temporaeres
Restore-Verzeichnis und reindiziert dort alle mbox-Dateien. Der
Identitaetsdigest muss vor und nach dem Reindex gleich bleiben.
Der lokale Snapshot ist nur die konsistente **Vorstufe** eines Backups. Er
enthaelt Kontozugangsdaten und muss anschliessend verschluesselt auf einen
raeumlich getrennten Datentraeger uebertragen werden.
## 6. Sicherheit
Tool hält fremde IMAP-Passwörter und kann Mails senden → Default bind
`127.0.0.1`. Falls remote nötig: hinter Caddy + `/vadmin`-Mail-2FA. `config.json`
und `*.db` nie committen.
Die Live-SQLite-Datei sowie ihre WAL-/SHM-Sidecars werden bei jedem Start auf
`0600` gesetzt. Das gilt auch fuer weitere `*.db`, `*.db-wal` und `*.db-shm`
direkt im konfigurierten Datenverzeichnis sowie fuer die Datenbank eines neu
erzeugten Snapshots. Kann ein Dateisystem Unix-Rechte nicht abbilden (z. B.
bestimmte exFAT-/CIFS-Mounts), startet das Werkzeug bewusst nicht: Zugangsdaten
bleiben fail-closed statt unbemerkt lesbar.
## 7. Reihenfolge bei einem echten Umzug
1. Ziel-Postfächer beim neuen Hoster anlegen (`archiv-…`, `temp-…`).

View file

@ -6,6 +6,8 @@ import (
"errors"
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
_ "modernc.org/sqlite"
@ -85,6 +87,10 @@ func ConnectDB(initDB bool) error {
_ = db.Close()
return err
}
if err := secureSQLiteFiles(Cfg.DBPath); err != nil {
_ = db.Close()
return err
}
schema := []string{
`CREATE TABLE IF NOT EXISTS accounts(
id INTEGER PRIMARY KEY AUTOINCREMENT,
@ -361,7 +367,7 @@ func ensureDBFile(path string, initDB bool) error {
if info.IsDir() {
return fmt.Errorf("DB %q ist ein Verzeichnis", path)
}
return nil
return chmodPrivateFile(path)
}
if !errors.Is(err, os.ErrNotExist) {
return err
@ -373,7 +379,57 @@ func ensureDBFile(path string, initDB bool) error {
if err != nil {
return err
}
return f.Close()
if err := f.Close(); err != nil {
return err
}
return chmodPrivateFile(path)
}
func secureSQLiteFiles(path string) error {
directory := filepath.Dir(path)
entries, err := os.ReadDir(directory)
if err != nil {
return fmt.Errorf("scan SQLite directory %q: %w", directory, err)
}
for _, entry := range entries {
if entry.Type()&os.ModeSymlink != 0 || !isSQLiteDataFile(entry.Name()) {
continue
}
info, err := entry.Info()
if err != nil {
return fmt.Errorf("inspect SQLite file %q: %w", filepath.Join(directory, entry.Name()), err)
}
if !info.Mode().IsRegular() {
continue
}
candidate := filepath.Join(directory, entry.Name())
if err := chmodPrivateFile(candidate); err != nil {
return fmt.Errorf("secure SQLite file %q: %w", candidate, err)
}
}
return nil
}
func chmodPrivateFile(path string) error {
if err := os.Chmod(path, 0o600); err != nil {
return err
}
if runtime.GOOS == "windows" {
return nil
}
info, err := os.Stat(path)
if err != nil {
return err
}
if permissions := info.Mode().Perm(); permissions&0o077 != 0 {
return fmt.Errorf("permissions=%#o after chmod 0600", permissions)
}
return nil
}
func isSQLiteDataFile(name string) bool {
name = strings.ToLower(name)
return strings.HasSuffix(name, ".db") || strings.HasSuffix(name, ".db-wal") || strings.HasSuffix(name, ".db-shm")
}
func ensureAppUserColumns() error {
@ -724,56 +780,72 @@ func GetCopyIdentityStateWithFolderAlias(accountID int64, folder, legacyFolder s
if legacyFolder != "" && legacyFolder != folder {
folders = append(folders, legacyFolder)
}
aliases := []string{identity.MessageID, identity.LegacyMessageID}
aliases = append(aliases, identity.LegacyMessageIDs...)
if identity.MessageID == "" && identity.BodySHA256 != "" {
aliases = append(aliases, "sha256:"+identity.BodySHA256)
}
uniqueAliases := make([]string, 0, len(aliases))
seenAliases := map[string]bool{}
for _, alias := range aliases {
alias = strings.TrimSpace(alias)
if alias == "" || seenAliases[alias] {
continue
}
seenAliases[alias] = true
uniqueAliases = append(uniqueAliases, alias)
}
// Precise rows always win. For Message-ID-less mail the hash is the whole
// identity, irrespective of which legacy key happened to be stored beside it.
// For mail with an ID, historical normalizations are precise aliases only
// when the canonical body hash is identical.
var mboxDone, targetDone int
for _, candidateFolder := range folders {
var mbox, target int
var err error
if identity.MessageID == "" {
err = DB.QueryRow(`SELECT COALESCE(MAX(mbox_done),0), COALESCE(MAX(target_done),0) FROM copied
var mbox, target int
err := DB.QueryRow(`SELECT COALESCE(MAX(mbox_done),0), COALESCE(MAX(target_done),0) FROM copied
WHERE account_id=? AND folder=? AND body_sha256=?`, accountID, candidateFolder, identity.BodySHA256).Scan(&mbox, &target)
} else {
err = DB.QueryRow(`SELECT COALESCE(MAX(mbox_done),0), COALESCE(MAX(target_done),0) FROM copied
if err != nil {
return CopyState{}, err
}
mboxDone |= mbox
targetDone |= target
continue
}
for _, alias := range uniqueAliases {
var mbox, target int
err := DB.QueryRow(`SELECT COALESCE(MAX(mbox_done),0), COALESCE(MAX(target_done),0) FROM copied
WHERE account_id=? AND folder=? AND message_id=? AND body_sha256=?`,
accountID, candidateFolder, identity.MessageID, identity.BodySHA256).Scan(&mbox, &target)
accountID, candidateFolder, alias, identity.BodySHA256).Scan(&mbox, &target)
if err != nil {
return CopyState{}, err
}
mboxDone |= mbox
targetDone |= target
}
if err != nil {
return CopyState{}, err
}
mboxDone |= mbox
targetDone |= target
}
if mboxDone != 0 || targetDone != 0 {
return CopyState{MboxDone: mboxDone != 0, TargetDone: targetDone != 0}, nil
}
// Once a Message-ID has precise archive identities, a different body with
// that same ID is new and must not be hidden by the old wildcard row.
if identity.MessageID != "" && identity.BodySHA256 != "" {
if len(uniqueAliases) != 0 && identity.BodySHA256 != "" {
var precise int
for _, candidateFolder := range folders {
var count int
if err := DB.QueryRow(`SELECT count(*) FROM copied WHERE account_id=? AND folder=? AND message_id=? AND body_sha256<>''`,
accountID, candidateFolder, identity.MessageID).Scan(&count); err != nil {
return CopyState{}, err
for _, alias := range uniqueAliases {
var count int
if err := DB.QueryRow(`SELECT count(*) FROM copied WHERE account_id=? AND folder=? AND message_id=? AND body_sha256<>''`,
accountID, candidateFolder, alias).Scan(&count); err != nil {
return CopyState{}, err
}
precise += count
}
precise += count
}
if precise > 0 {
return CopyState{}, nil
}
}
aliases := []string{identity.MessageID, identity.LegacyMessageID}
if identity.MessageID == "" && identity.BodySHA256 != "" {
aliases = append(aliases, "sha256:"+identity.BodySHA256)
}
seen := map[string]bool{}
for _, alias := range aliases {
alias = strings.TrimSpace(alias)
if alias == "" || seen[alias] {
continue
}
seen[alias] = true
for _, alias := range uniqueAliases {
for _, candidateFolder := range folders {
var mbox, target int
err := DB.QueryRow(`SELECT COALESCE(MAX(mbox_done),0), COALESCE(MAX(target_done),0) FROM copied
@ -875,21 +947,25 @@ func ReplaceMboxIndex(accountID int64, folder string, entries []MboxIndexEntry,
if err := aliasRows.Close(); err != nil {
return err
}
legacyIDsBySeq := map[int]string{}
type storedIndexIdentity struct {
messageID string
bodySHA256 string
}
storedIdentityBySeq := map[int]storedIndexIdentity{}
for _, candidateFolder := range folderAliases {
rows, err := tx.Query(`SELECT seq, message_id FROM mbox_index WHERE account_id=? AND folder=?`, accountID, candidateFolder)
rows, err := tx.Query(`SELECT seq, message_id, body_sha256 FROM mbox_index WHERE account_id=? AND folder=?`, accountID, candidateFolder)
if err != nil {
return err
}
for rows.Next() {
var seq int
var messageID string
if err := rows.Scan(&seq, &messageID); err != nil {
var identity storedIndexIdentity
if err := rows.Scan(&seq, &identity.messageID, &identity.bodySHA256); err != nil {
_ = rows.Close()
return err
}
if _, exists := legacyIDsBySeq[seq]; !exists || candidateFolder == folder {
legacyIDsBySeq[seq] = messageID
if _, exists := storedIdentityBySeq[seq]; !exists || candidateFolder == folder {
storedIdentityBySeq[seq] = identity
}
}
if err := rows.Err(); err != nil {
@ -907,6 +983,15 @@ func ReplaceMboxIndex(accountID int64, folder string, entries []MboxIndexEntry,
entry.AccountID = accountID
entry.Folder = folder
entry.Seq = i
storedIdentity := storedIdentityBySeq[i]
// The IMAP envelope and a later RFC-822 header parse can represent the
// same malformed Message-ID differently. If the canonical body at the
// same archive position is identical, retain the already committed ID so
// a rebuild is an exact metadata restore. Never carry it across a body
// change, because that could attach an old identity to different content.
if storedIdentity.bodySHA256 != "" && storedIdentity.bodySHA256 == entry.BodySHA256 {
entry.MessageID = storedIdentity.messageID
}
if entry.MessageID == "" && entry.BodySHA256 == "" {
continue
}
@ -921,7 +1006,7 @@ func ReplaceMboxIndex(accountID int64, folder string, entries []MboxIndexEntry,
// pipeline the target append happened before the mbox append, so recovered
// records are complete in both stages. Marking them prevents a later watch
// run from appending duplicates to either destination.
state, found, err := copyStateForReindex(tx, entry.AccountID, folderAliases, entry, legacyIDsBySeq[i])
state, found, err := copyStateForReindex(tx, entry.AccountID, folderAliases, entry, storedIdentity.messageID)
if err != nil {
return err
}

View file

@ -31,10 +31,13 @@ type RawMessage struct {
// for byte-different messages. Messages without a Message-ID are identified by
// BodySHA256 alone. LegacyMessageID keeps the old raw-IMAP hash addressable
// while existing databases are migrated without re-copying mail.
// LegacyMessageIDs additionally retain historical Message-ID normalizations
// from both the IMAP envelope and the raw RFC-822 header.
type MessageIdentity struct {
MessageID string
BodySHA256 string
LegacyMessageID string
MessageID string
BodySHA256 string
LegacyMessageID string
LegacyMessageIDs []string
}
type MessageHeader struct {
@ -489,19 +492,23 @@ func messageID(body []byte) string {
func identityForRawMessage(m RawMessage) MessageIdentity {
id := normalizeMessageID(m.MessageID)
legacyID := id
legacyID := legacyNormalizeMessageID(m.MessageID)
aliases := []string{legacyID}
if strings.HasPrefix(strings.ToLower(id), "sha256:") {
id = ""
}
if id == "" {
if msg, err := mail.ReadMessage(bytes.NewReader(m.Body)); err == nil {
id = normalizeMessageID(msg.Header.Get("Message-ID"))
if msg, err := mail.ReadMessage(bytes.NewReader(m.Body)); err == nil {
headerID := msg.Header.Get("Message-ID")
aliases = append(aliases, normalizeMessageID(headerID), legacyNormalizeMessageID(headerID))
if id == "" {
id = normalizeMessageID(headerID)
}
}
return MessageIdentity{
MessageID: id,
BodySHA256: bodySHA256(m.Body),
LegacyMessageID: legacyID,
MessageID: id,
BodySHA256: bodySHA256(m.Body),
LegacyMessageID: legacyID,
LegacyMessageIDs: aliases,
}
}
@ -525,6 +532,18 @@ func bodySHA256(raw []byte) string {
}
func normalizeMessageID(id string) string {
id = strings.TrimSpace(id)
if start := strings.IndexByte(id, '<'); start >= 0 {
if relativeEnd := strings.IndexByte(id[start+1:], '>'); relativeEnd >= 0 {
if candidate := strings.TrimSpace(id[start+1 : start+1+relativeEnd]); candidate != "" {
return candidate
}
}
}
return strings.Trim(id, "<>")
}
func legacyNormalizeMessageID(id string) string {
return strings.Trim(strings.TrimSpace(id), "<>")
}

View file

@ -25,6 +25,56 @@ func TestMessageIDFallsBackToBodyHash(t *testing.T) {
}
}
func TestNormalizeMessageIDExtractsBracketedIDBeforeBrokenComment(t *testing.T) {
tests := map[string]string{
"<normal@example.com>": "normal@example.com",
"bracketless@example.com": "bracketless@example.com",
" <5486CB7400AF1FB7@mr001msb.fastweb.it> (added by postmaster) ": "5486CB7400AF1FB7@mr001msb.fastweb.it",
"comment <embedded@example.com> trailing": "embedded@example.com",
}
for input, want := range tests {
if got := normalizeMessageID(input); got != want {
t.Errorf("normalizeMessageID(%q)=%q, want %q", input, got, want)
}
}
}
func TestIdentityKeepsOldMalformedHeaderNormalizationAsAlias(t *testing.T) {
tests := []struct {
header string
clean string
legacy string
}{
{
header: `<476BDFDB019289BE@mail21.bluewin.ch> (added by postmaster@bluewin.ch)`,
clean: `476BDFDB019289BE@mail21.bluewin.ch`,
legacy: `476BDFDB019289BE@mail21.bluewin.ch> (added by postmaster@bluewin.ch)`,
},
{
header: `<22578196653698419217.BB8A0E8F48F3F290@dr-gold.de>+D271B1409C3C6028`,
clean: `22578196653698419217.BB8A0E8F48F3F290@dr-gold.de`,
legacy: `22578196653698419217.BB8A0E8F48F3F290@dr-gold.de>+D271B1409C3C6028`,
},
}
for _, tt := range tests {
body := []byte("Message-ID: " + tt.header + "\r\nSubject: malformed ID\r\n\r\nbody")
identity := identityForRawMessage(RawMessage{MessageID: messageID(body), Body: body})
if identity.MessageID != tt.clean {
t.Fatalf("MessageID=%q, want %q", identity.MessageID, tt.clean)
}
found := false
for _, alias := range identity.LegacyMessageIDs {
if alias == tt.legacy {
found = true
break
}
}
if !found {
t.Fatalf("legacy alias %q missing from %#v", tt.legacy, identity.LegacyMessageIDs)
}
}
}
func TestBodySHA256IsStableAcrossMboxNormalization(t *testing.T) {
raw := []byte("From: a@example.com\r\nSubject: Test\r\n\r\nFirst\r\nFrom escaped\r\n\r\n")
record := mboxRecord(RawMessage{Body: raw})

View file

@ -1,6 +1,7 @@
package backend
import (
"bufio"
"bytes"
"encoding/base64"
"fmt"
@ -290,17 +291,55 @@ func reindexAccountArchive(account Account) error {
}
func reindexPlainMbox(path string) ([]MboxIndexEntry, error) {
b, err := os.ReadFile(path)
f, err := os.Open(path)
if err != nil {
return nil, err
}
defer f.Close()
reader := bufio.NewReaderSize(f, 256*1024)
var out []MboxIndexEntry
for _, part := range splitMboxRecordsWithOffsets(b) {
entry := indexEntryFromMboxRecord(part.Message, part.Offset, int64(part.Length), 0, int64(part.Length))
var record bytes.Buffer
var offset int64
var messageStart int64 = -1
appendRecord := func(end int64) {
if messageStart < 0 || end <= messageStart {
return
}
message := unescapeMboxMessage(record.Bytes())
if len(message) == 0 {
return
}
entry := indexEntryFromMboxRecord(message, messageStart, end-messageStart, 0, end-messageStart)
if entry.MessageID != "" || entry.BodySHA256 != "" {
out = append(out, entry)
}
}
for {
lineStart := offset
line, readErr := reader.ReadBytes('\n')
offset += int64(len(line))
content := line
if len(content) > 0 && content[len(content)-1] == '\n' {
content = content[:len(content)-1]
}
if len(content) > 0 && content[len(content)-1] == '\r' {
content = content[:len(content)-1]
}
if bytes.HasPrefix(content, []byte("From ")) {
appendRecord(lineStart)
record = bytes.Buffer{}
messageStart = offset
} else if messageStart >= 0 && len(line) > 0 {
_, _ = record.Write(line)
}
if readErr != nil {
if readErr != io.EOF {
return nil, readErr
}
break
}
}
appendRecord(offset)
return out, nil
}

View file

@ -3,8 +3,10 @@ package backend
import (
"bytes"
"database/sql"
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
@ -12,6 +14,39 @@ import (
"github.com/klauspost/compress/zstd"
)
func TestReindexPlainMboxStreamingMatchesReferenceParser(t *testing.T) {
var mbox bytes.Buffer
for i := 0; i < 257; i++ {
body := strings.Repeat(fmt.Sprintf("line-%03d From inside body\r\n", i), i%31+1)
mbox.Write(mboxRecord(RawMessage{
MessageID: fmt.Sprintf("stream-%03d@example.com", i),
Body: []byte(fmt.Sprintf(
"Message-ID: <stream-%03d@example.com>\r\nSubject: Stream %03d\r\n\r\n%s",
i, i, body,
)),
}))
}
path := filepath.Join(t.TempDir(), "streaming.mbox")
if err := os.WriteFile(path, mbox.Bytes(), 0o600); err != nil {
t.Fatal(err)
}
got, err := reindexPlainMbox(path)
if err != nil {
t.Fatal(err)
}
parts := splitMboxRecordsWithOffsets(mbox.Bytes())
if len(got) != len(parts) {
t.Fatalf("streaming entries=%d, reference parts=%d", len(got), len(parts))
}
for i, part := range parts {
want := indexEntryFromMboxRecord(part.Message, part.Offset, int64(part.Length), 0, int64(part.Length))
if got[i] != want {
t.Fatalf("entry %d differs:\n got %#v\n want %#v", i, got[i], want)
}
}
}
func TestZstdMboxRoundTripAndIndexRead(t *testing.T) {
oldCfg, oldDB := Cfg, DB
root := t.TempDir()
@ -325,6 +360,14 @@ func TestConnectDBConfiguresBusyTimeoutAndCopyStages(t *testing.T) {
Cfg, DB = oldCfg, oldDB
})
Cfg = Config{DBPath: filepath.Join(root, "mail-graveyard.db"), MboxRoot: filepath.Join(root, "backup")}
backupDB := filepath.Join(root, "PRE-existing-copy.db")
if err := os.WriteFile(backupDB, []byte("diagnostic copy"), 0o644); err != nil {
t.Fatal(err)
}
nonDB := filepath.Join(root, "keep-mode.txt")
if err := os.WriteFile(nonDB, []byte("not a database"), 0o644); err != nil {
t.Fatal(err)
}
if err := ConnectDB(true); err != nil {
t.Fatal(err)
}
@ -335,6 +378,24 @@ func TestConnectDBConfiguresBusyTimeoutAndCopyStages(t *testing.T) {
if timeout != 10000 {
t.Fatalf("busy_timeout=%d, want 10000", timeout)
}
if runtime.GOOS != "windows" {
for _, path := range []string{Cfg.DBPath, Cfg.DBPath + "-wal", Cfg.DBPath + "-shm", backupDB} {
info, err := os.Stat(path)
if err != nil {
t.Fatal(err)
}
if permissions := info.Mode().Perm(); permissions != 0o600 {
t.Fatalf("SQLite file %s permissions=%#o, want 0600", path, permissions)
}
}
info, err := os.Stat(nonDB)
if err != nil {
t.Fatal(err)
}
if permissions := info.Mode().Perm(); permissions != 0o644 {
t.Fatalf("non-DB file permissions changed to %#o", permissions)
}
}
if err := SaveAccount(Account{
Name: "stage-account", SrcHost: "source.example", SrcPort: 993, SrcSecurity: "tls", SrcUser: "source@example.com", SrcPass: "x",
DstHost: "target.example", DstPort: 993, DstSecurity: "tls", DstUser: "target@example.com", DstPass: "x", Active: true,
@ -379,6 +440,106 @@ func TestConnectDBConfiguresBusyTimeoutAndCopyStages(t *testing.T) {
}
}
func TestReplaceMboxIndexPreservesStoredMessageIDOnlyForSameBody(t *testing.T) {
oldCfg, oldDB := Cfg, DB
root := t.TempDir()
t.Cleanup(func() {
if DB != nil {
_ = DB.Close()
}
Cfg, DB = oldCfg, oldDB
})
Cfg = Config{DBPath: filepath.Join(root, "mail-graveyard.db"), MboxRoot: filepath.Join(root, "backup")}
if err := ConnectDB(true); err != nil {
t.Fatal(err)
}
if err := SaveAccount(Account{
Name: "identity-restore", SrcHost: "source.example", SrcPort: 993, SrcSecurity: "tls", SrcUser: "source@example.com", SrcPass: "x",
DstHost: "target.example", DstPort: 993, DstSecurity: "tls", DstUser: "target@example.com", DstPass: "x", Active: true,
}); err != nil {
t.Fatal(err)
}
account, err := GetAccount("identity-restore")
if err != nil {
t.Fatal(err)
}
stableHash := bodySHA256([]byte("same archived body"))
if err := ReplaceMboxIndex(account.ID, "INBOX", []MboxIndexEntry{{
MessageID: "stored@example.com> (added by postmaster)", BodySHA256: stableHash, FrameLen: 10, InnerLen: 10,
}}, 10); err != nil {
t.Fatal(err)
}
if err := ReplaceMboxIndex(account.ID, "INBOX", []MboxIndexEntry{{
MessageID: "parsed@example.com", BodySHA256: stableHash, FrameLen: 10, InnerLen: 10,
}}, 10); err != nil {
t.Fatal(err)
}
entry, err := GetMboxIndex(account.ID, "INBOX", 0)
if err != nil {
t.Fatal(err)
}
if entry.MessageID != "stored@example.com> (added by postmaster)" {
t.Fatalf("same body lost stored Message-ID: %q", entry.MessageID)
}
changedHash := bodySHA256([]byte("different archived body"))
if err := ReplaceMboxIndex(account.ID, "INBOX", []MboxIndexEntry{{
MessageID: "new-content@example.com", BodySHA256: changedHash, FrameLen: 11, InnerLen: 11,
}}, 11); err != nil {
t.Fatal(err)
}
entry, err = GetMboxIndex(account.ID, "INBOX", 0)
if err != nil {
t.Fatal(err)
}
if entry.MessageID != "new-content@example.com" {
t.Fatalf("different body inherited stale Message-ID: %q", entry.MessageID)
}
}
func TestPreciseLegacyMessageIDAliasesPreventLiveRecopy(t *testing.T) {
account, _, restore := setupSnapshotTest(t, "none")
defer restore()
tests := []struct {
header string
legacy string
}{
{
header: `<476BDFDB019289BE@mail21.bluewin.ch> (added by postmaster@bluewin.ch)`,
legacy: `476BDFDB019289BE@mail21.bluewin.ch> (added by postmaster@bluewin.ch)`,
},
{
header: `<22578196653698419217.BB8A0E8F48F3F290@dr-gold.de>+D271B1409C3C6028`,
legacy: `22578196653698419217.BB8A0E8F48F3F290@dr-gold.de>+D271B1409C3C6028`,
},
}
for i, tt := range tests {
folder := fmt.Sprintf("INBOX.%d", i)
body := []byte("Message-ID: " + tt.header + "\r\nSubject: malformed ID\r\n\r\noriginal body")
identity := identityForRawMessage(RawMessage{MessageID: messageID(body), Body: body})
if err := MarkIdentityCopied(account.ID, folder, MessageIdentity{MessageID: tt.legacy, BodySHA256: identity.BodySHA256}); err != nil {
t.Fatal(err)
}
state, err := GetCopyIdentityState(account.ID, folder, identity)
if err != nil {
t.Fatal(err)
}
if !state.MboxDone || !state.TargetDone {
t.Fatalf("fixture %d precise legacy alias was not recognized: %#v", i, state)
}
changedBody := []byte("Message-ID: " + tt.header + "\r\nSubject: malformed ID\r\n\r\ndifferent body")
changedIdentity := identityForRawMessage(RawMessage{MessageID: messageID(changedBody), Body: changedBody})
state, err = GetCopyIdentityState(account.ID, folder, changedIdentity)
if err != nil {
t.Fatal(err)
}
if state.MboxDone || state.TargetDone {
t.Fatalf("fixture %d legacy alias hid byte-different mail: %#v", i, state)
}
}
}
func TestIdentitySchemaUpgradePreservesLegacyAliases(t *testing.T) {
oldCfg, oldDB := Cfg, DB
root := t.TempDir()

711
backend/14-snapshot.go Normal file
View file

@ -0,0 +1,711 @@
package backend
import (
"crypto/sha256"
"database/sql"
"encoding/binary"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"log"
"os"
"path/filepath"
"runtime"
"sort"
"strings"
"time"
)
const snapshotFormatVersion = 1
type SnapshotManifest struct {
FormatVersion int `json:"format_version"`
CreatedAt string `json:"created_at"`
DatabasePath string `json:"database_path"`
DatabaseSHA256 string `json:"database_sha256"`
IndexSHA256 string `json:"index_sha256"`
IndexRecords int64 `json:"index_records"`
PreciseCopied int64 `json:"precise_copied"`
PendingMbox int64 `json:"pending_mbox"`
PendingTarget int64 `json:"pending_target"`
InvalidStageOrder int64 `json:"invalid_stage_order"`
MissingArchiveRecords int64 `json:"missing_archive_records"`
Files []SnapshotFile `json:"files"`
}
type SnapshotFile struct {
Path string `json:"path"`
Bytes int64 `json:"bytes"`
SHA256 string `json:"sha256"`
AccountID int64 `json:"account_id"`
Account string `json:"account"`
Folder string `json:"folder"`
IndexRecords int64 `json:"index_records"`
}
type snapshotWatermark struct {
AccountID int64
Account string
MboxDir string
Folder string
Bytes int64
IndexRecords int64
SourcePath string
RelativePath string
}
// CreateSnapshot creates a self-consistent DB + mbox-prefix snapshot without
// stopping append-only archive writers. The DB snapshot is taken first; every
// copied mbox prefix is then bounded exclusively by offsets from that snapshot.
func CreateSnapshot(target string) error {
return createSnapshot(target, nil)
}
func createSnapshot(target string, afterDatabaseSnapshot func() error) (retErr error) {
if DB == nil {
return fmt.Errorf("snapshot: database is not connected")
}
target = strings.TrimSpace(target)
if target == "" {
return fmt.Errorf("snapshot: target directory is empty")
}
targetAbs, err := filepath.Abs(target)
if err != nil {
return fmt.Errorf("snapshot target: %w", err)
}
if _, err := os.Stat(targetAbs); err == nil {
return fmt.Errorf("snapshot target already exists: %s", targetAbs)
} else if !os.IsNotExist(err) {
return fmt.Errorf("snapshot target stat: %w", err)
}
if err := os.MkdirAll(filepath.Dir(targetAbs), 0o700); err != nil {
return fmt.Errorf("snapshot target parent: %w", err)
}
partial := fmt.Sprintf("%s.partial-%d-%d", targetAbs, os.Getpid(), time.Now().UnixNano())
if err := os.Mkdir(partial, 0o700); err != nil {
return fmt.Errorf("snapshot staging directory: %w", err)
}
complete := false
defer func() {
if !complete {
log.Printf("snapshot incomplete; staging data retained at %s", partial)
if retErr != nil {
retErr = fmt.Errorf("%w (incomplete snapshot retained at %s)", retErr, partial)
}
}
}()
databaseRel := filepath.ToSlash(filepath.Join("data", "mail-graveyard.db"))
databasePath := filepath.Join(partial, filepath.FromSlash(databaseRel))
if err := os.MkdirAll(filepath.Dir(databasePath), 0o700); err != nil {
return fmt.Errorf("snapshot database directory: %w", err)
}
if _, err := DB.Exec(`VACUUM INTO ?`, databasePath); err != nil {
return fmt.Errorf("snapshot database VACUUM INTO: %w", err)
}
if err := chmodPrivateFile(databasePath); err != nil {
return fmt.Errorf("snapshot database permissions: %w", err)
}
if err := syncFile(databasePath); err != nil {
return fmt.Errorf("snapshot database sync: %w", err)
}
if afterDatabaseSnapshot != nil {
if err := afterDatabaseSnapshot(); err != nil {
return fmt.Errorf("snapshot test hook: %w", err)
}
}
snapshotDB, err := openSnapshotDatabase(databasePath)
if err != nil {
return err
}
watermarks, err := collectSnapshotWatermarks(snapshotDB, Cfg.MboxRoot)
if err != nil {
_ = snapshotDB.Close()
return err
}
stats, err := snapshotDatabaseStats(snapshotDB)
if err == nil {
stats.IndexSHA256, err = indexDigest(snapshotDB)
}
if closeErr := snapshotDB.Close(); err == nil && closeErr != nil {
err = closeErr
}
if err != nil {
return err
}
if stats.MissingArchiveRecords != 0 {
return fmt.Errorf("snapshot refused: %d mbox_done identities have no archive index", stats.MissingArchiveRecords)
}
if stats.InvalidStageOrder != 0 {
return fmt.Errorf("snapshot refused: %d precise identities have target_done=1 before mbox_done", stats.InvalidStageOrder)
}
var watermarkRecords int64
for _, watermark := range watermarks {
watermarkRecords += watermark.IndexRecords
}
if watermarkRecords != stats.IndexRecords {
return fmt.Errorf("snapshot refused: indexed records=%d but mapped mbox records=%d", stats.IndexRecords, watermarkRecords)
}
manifest := SnapshotManifest{
FormatVersion: snapshotFormatVersion,
CreatedAt: time.Now().UTC().Format(time.RFC3339Nano),
DatabasePath: databaseRel,
IndexSHA256: stats.IndexSHA256,
IndexRecords: stats.IndexRecords,
PreciseCopied: stats.PreciseCopied,
PendingMbox: stats.PendingMbox,
PendingTarget: stats.PendingTarget,
InvalidStageOrder: stats.InvalidStageOrder,
MissingArchiveRecords: stats.MissingArchiveRecords,
Files: make([]SnapshotFile, 0, len(watermarks)),
}
manifest.DatabaseSHA256, err = hashFile(databasePath)
if err != nil {
return fmt.Errorf("snapshot database hash: %w", err)
}
for _, watermark := range watermarks {
destinationRel := filepath.ToSlash(filepath.Join("backup", watermark.RelativePath))
destination := filepath.Join(partial, filepath.FromSlash(destinationRel))
hash, err := copyFilePrefix(watermark.SourcePath, destination, watermark.Bytes)
if err != nil {
return fmt.Errorf("snapshot mbox %s: %w", destinationRel, err)
}
manifest.Files = append(manifest.Files, SnapshotFile{
Path: destinationRel,
Bytes: watermark.Bytes,
SHA256: hash,
AccountID: watermark.AccountID,
Account: watermark.Account,
Folder: watermark.Folder,
IndexRecords: watermark.IndexRecords,
})
}
sortSnapshotFiles(manifest.Files)
manifestPath := filepath.Join(partial, "snapshot-manifest.json")
manifestJSON, err := json.MarshalIndent(manifest, "", " ")
if err != nil {
return fmt.Errorf("snapshot manifest encode: %w", err)
}
manifestJSON = append(manifestJSON, '\n')
if err := os.WriteFile(manifestPath, manifestJSON, 0o600); err != nil {
return fmt.Errorf("snapshot manifest write: %w", err)
}
if err := syncFile(manifestPath); err != nil {
return fmt.Errorf("snapshot manifest sync: %w", err)
}
if err := os.Rename(partial, targetAbs); err != nil {
return fmt.Errorf("snapshot finalize: %w", err)
}
complete = true
log.Printf("snapshot complete target=%s files=%d bytes=%d index_records=%d precise_copied=%d pending_mbox=%d pending_target=%d",
targetAbs, len(manifest.Files), snapshotBytes(manifest.Files), manifest.IndexRecords, manifest.PreciseCopied, manifest.PendingMbox, manifest.PendingTarget)
return nil
}
type snapshotStats struct {
IndexRecords int64
PreciseCopied int64
PendingMbox int64
PendingTarget int64
MissingArchiveRecords int64
InvalidStageOrder int64
IndexSHA256 string
}
func snapshotDatabaseStats(db *sql.DB) (snapshotStats, error) {
var stats snapshotStats
queries := []struct {
query string
dest *int64
}{
{`SELECT count(*) FROM mbox_index`, &stats.IndexRecords},
{`SELECT count(*) FROM copied WHERE body_sha256<>''`, &stats.PreciseCopied},
{`SELECT count(*) FROM copied WHERE body_sha256<>'' AND mbox_done=0`, &stats.PendingMbox},
{`SELECT count(*) FROM copied WHERE body_sha256<>'' AND target_done=0`, &stats.PendingTarget},
{`SELECT count(*) FROM copied WHERE body_sha256<>'' AND target_done=1 AND mbox_done=0`, &stats.InvalidStageOrder},
{`SELECT count(*) FROM copied c WHERE c.body_sha256<>'' AND c.mbox_done=1 AND NOT EXISTS (
SELECT 1 FROM mbox_index i
WHERE i.account_id=c.account_id AND i.folder=c.folder
AND i.message_id=c.message_id AND i.body_sha256=c.body_sha256
)`, &stats.MissingArchiveRecords},
}
for _, item := range queries {
if err := db.QueryRow(item.query).Scan(item.dest); err != nil {
return snapshotStats{}, fmt.Errorf("snapshot database statistics: %w", err)
}
}
return stats, nil
}
func indexDigest(db *sql.DB) (string, error) {
rows, err := db.Query(`SELECT account_id, folder, seq, message_id, body_sha256
FROM mbox_index ORDER BY account_id, folder, seq`)
if err != nil {
return "", fmt.Errorf("snapshot index digest query: %w", err)
}
defer rows.Close()
hash := sha256.New()
for rows.Next() {
var accountID, seq int64
var folder, messageID, bodyHash string
if err := rows.Scan(&accountID, &folder, &seq, &messageID, &bodyHash); err != nil {
return "", fmt.Errorf("snapshot index digest row: %w", err)
}
for _, value := range []int64{accountID, seq} {
if err := binary.Write(hash, binary.LittleEndian, value); err != nil {
return "", err
}
}
for _, value := range []string{folder, messageID, bodyHash} {
if err := binary.Write(hash, binary.LittleEndian, uint64(len(value))); err != nil {
return "", err
}
if _, err := io.WriteString(hash, value); err != nil {
return "", err
}
}
}
if err := rows.Err(); err != nil {
return "", fmt.Errorf("snapshot index digest: %w", err)
}
return hex.EncodeToString(hash.Sum(nil)), nil
}
func collectSnapshotWatermarks(db *sql.DB, mboxRoot string) ([]snapshotWatermark, error) {
rootAbs, err := filepath.Abs(mboxRoot)
if err != nil {
return nil, fmt.Errorf("snapshot mbox root: %w", err)
}
rows, err := db.Query(`SELECT i.account_id, a.name, a.mbox_dir, i.folder,
MAX(i.file_offset+i.frame_len), count(*)
FROM mbox_index i JOIN accounts a ON a.id=i.account_id
GROUP BY i.account_id, a.name, a.mbox_dir, i.folder
ORDER BY lower(a.name), a.name, i.folder`)
if err != nil {
return nil, fmt.Errorf("snapshot watermarks: %w", err)
}
defer rows.Close()
var out []snapshotWatermark
seenPaths := map[string]bool{}
archivePaths := map[string]map[string]string{}
for rows.Next() {
var watermark snapshotWatermark
if err := rows.Scan(&watermark.AccountID, &watermark.Account, &watermark.MboxDir, &watermark.Folder, &watermark.Bytes, &watermark.IndexRecords); err != nil {
return nil, fmt.Errorf("snapshot watermark row: %w", err)
}
watermark.MboxDir = strings.TrimSpace(watermark.MboxDir)
if watermark.MboxDir == "" {
return nil, fmt.Errorf("snapshot account %s has no mbox_dir", watermark.Account)
}
paths, ok := archivePaths[watermark.MboxDir]
if !ok {
paths = map[string]string{}
for _, path := range archiveMboxPaths(filepath.Join(rootAbs, watermark.MboxDir)) {
paths[folderNameFromMboxPath(path)] = path
}
archivePaths[watermark.MboxDir] = paths
}
watermark.SourcePath = paths[watermark.Folder]
if watermark.SourcePath == "" {
return nil, fmt.Errorf("snapshot indexed mbox missing for account=%s folder=%q", watermark.Account, watermark.Folder)
}
pathAbs, err := filepath.Abs(watermark.SourcePath)
if err != nil {
return nil, fmt.Errorf("snapshot mbox path: %w", err)
}
relative, err := filepath.Rel(rootAbs, pathAbs)
if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(os.PathSeparator)) || filepath.IsAbs(relative) {
return nil, fmt.Errorf("snapshot mbox escapes configured root: %s", pathAbs)
}
watermark.SourcePath = pathAbs
watermark.RelativePath = relative
pathKey := snapshotPathKey(relative)
if seenPaths[pathKey] {
return nil, fmt.Errorf("snapshot multiple index groups resolve to the same mbox: %s", relative)
}
seenPaths[pathKey] = true
info, err := os.Stat(pathAbs)
if err != nil {
return nil, fmt.Errorf("snapshot mbox stat %s: %w", pathAbs, err)
}
if !info.Mode().IsRegular() {
return nil, fmt.Errorf("snapshot mbox is not a regular file: %s", pathAbs)
}
if watermark.Bytes <= 0 || info.Size() < watermark.Bytes {
return nil, fmt.Errorf("snapshot invalid watermark account=%s folder=%q boundary=%d size=%d",
watermark.Account, watermark.Folder, watermark.Bytes, info.Size())
}
out = append(out, watermark)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("snapshot watermarks: %w", err)
}
return out, nil
}
func copyFilePrefix(source, destination string, length int64) (string, error) {
if length < 0 {
return "", fmt.Errorf("negative prefix length %d", length)
}
if err := os.MkdirAll(filepath.Dir(destination), 0o700); err != nil {
return "", err
}
src, err := os.Open(source)
if err != nil {
return "", err
}
defer src.Close()
dst, err := os.OpenFile(destination, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
if err != nil {
return "", err
}
hash := sha256.New()
written, copyErr := io.CopyN(io.MultiWriter(dst, hash), src, length)
if copyErr == nil && written != length {
copyErr = io.ErrShortWrite
}
if copyErr == nil {
copyErr = dst.Sync()
}
closeErr := dst.Close()
if copyErr != nil {
return "", copyErr
}
if closeErr != nil {
return "", closeErr
}
return hex.EncodeToString(hash.Sum(nil)), nil
}
func openSnapshotDatabase(path string) (*sql.DB, error) {
db, err := sql.Open("sqlite", path)
if err != nil {
return nil, fmt.Errorf("open snapshot database: %w", err)
}
db.SetMaxOpenConns(1)
if _, err := db.Exec(`PRAGMA query_only=ON; PRAGMA foreign_keys=ON;`); err != nil {
_ = db.Close()
return nil, fmt.Errorf("open snapshot database read-only: %w", err)
}
return db, nil
}
// VerifySnapshot validates hashes and DB/file boundaries, then copies only the
// small DB into a temporary restore workspace and performs a full reindex there.
// The snapshot mbox files and any live database remain untouched.
func VerifySnapshot(root string) error {
root = strings.TrimSpace(root)
if root == "" {
return fmt.Errorf("verify snapshot: directory is empty")
}
rootAbs, err := filepath.Abs(root)
if err != nil {
return fmt.Errorf("verify snapshot path: %w", err)
}
manifestBytes, err := os.ReadFile(filepath.Join(rootAbs, "snapshot-manifest.json"))
if err != nil {
return fmt.Errorf("verify snapshot manifest: %w", err)
}
var manifest SnapshotManifest
if err := json.Unmarshal(manifestBytes, &manifest); err != nil {
return fmt.Errorf("verify snapshot manifest decode: %w", err)
}
if manifest.FormatVersion != snapshotFormatVersion {
return fmt.Errorf("verify snapshot format=%d, supported=%d", manifest.FormatVersion, snapshotFormatVersion)
}
databasePath, err := pathInsideSnapshot(rootAbs, manifest.DatabasePath)
if err != nil {
return err
}
if runtime.GOOS != "windows" {
databaseInfo, err := os.Stat(databasePath)
if err != nil {
return fmt.Errorf("verify snapshot database permissions: %w", err)
}
if permissions := databaseInfo.Mode().Perm(); permissions&0o077 != 0 {
return fmt.Errorf("verify snapshot database permissions=%#o, want no group/other access", permissions)
}
}
databaseHash, err := hashFile(databasePath)
if err != nil {
return fmt.Errorf("verify snapshot database hash: %w", err)
}
if databaseHash != manifest.DatabaseSHA256 {
return fmt.Errorf("verify snapshot database hash mismatch: got %s want %s", databaseHash, manifest.DatabaseSHA256)
}
for _, file := range manifest.Files {
path, err := pathInsideSnapshot(rootAbs, file.Path)
if err != nil {
return err
}
info, err := os.Stat(path)
if err != nil {
return fmt.Errorf("verify snapshot file %s: %w", file.Path, err)
}
if info.Size() != file.Bytes {
return fmt.Errorf("verify snapshot file size %s: got %d want %d", file.Path, info.Size(), file.Bytes)
}
hash, err := hashFile(path)
if err != nil {
return fmt.Errorf("verify snapshot file hash %s: %w", file.Path, err)
}
if hash != file.SHA256 {
return fmt.Errorf("verify snapshot file hash mismatch %s: got %s want %s", file.Path, hash, file.SHA256)
}
}
snapshotDB, err := openSnapshotDatabase(databasePath)
if err != nil {
return err
}
var integrity string
if err := snapshotDB.QueryRow(`PRAGMA integrity_check`).Scan(&integrity); err != nil {
_ = snapshotDB.Close()
return fmt.Errorf("verify snapshot integrity: %w", err)
}
if integrity != "ok" {
_ = snapshotDB.Close()
return fmt.Errorf("verify snapshot integrity: %s", integrity)
}
stats, err := snapshotDatabaseStats(snapshotDB)
if err == nil {
stats.IndexSHA256, err = indexDigest(snapshotDB)
}
if err != nil {
_ = snapshotDB.Close()
return err
}
watermarks, err := collectSnapshotWatermarks(snapshotDB, filepath.Join(rootAbs, "backup"))
if closeErr := snapshotDB.Close(); err == nil && closeErr != nil {
err = closeErr
}
if err != nil {
return err
}
if stats.IndexRecords != manifest.IndexRecords || stats.PreciseCopied != manifest.PreciseCopied ||
stats.PendingMbox != manifest.PendingMbox || stats.PendingTarget != manifest.PendingTarget ||
stats.InvalidStageOrder != manifest.InvalidStageOrder || stats.MissingArchiveRecords != manifest.MissingArchiveRecords ||
stats.IndexSHA256 != manifest.IndexSHA256 {
return fmt.Errorf("verify snapshot manifest/database statistics differ")
}
if stats.MissingArchiveRecords != 0 {
return fmt.Errorf("verify snapshot has %d mbox_done identities without archive index", stats.MissingArchiveRecords)
}
if stats.InvalidStageOrder != 0 {
return fmt.Errorf("verify snapshot has %d precise identities with invalid stage order", stats.InvalidStageOrder)
}
if err := compareManifestWatermarks(manifest.Files, watermarks); err != nil {
return err
}
if err := verifySnapshotReindex(rootAbs, databasePath, manifest.IndexRecords, manifest.IndexSHA256); err != nil {
return err
}
log.Printf("snapshot verified root=%s files=%d bytes=%d index_records=%d precise_copied=%d pending_mbox=%d pending_target=%d",
rootAbs, len(manifest.Files), snapshotBytes(manifest.Files), manifest.IndexRecords, manifest.PreciseCopied, manifest.PendingMbox, manifest.PendingTarget)
return nil
}
func compareManifestWatermarks(files []SnapshotFile, watermarks []snapshotWatermark) error {
manifestByPath := make(map[string]SnapshotFile, len(files))
for _, file := range files {
key := snapshotPathKey(filepath.FromSlash(file.Path))
if _, exists := manifestByPath[key]; exists {
return fmt.Errorf("verify snapshot duplicate manifest path: %s", file.Path)
}
manifestByPath[key] = file
}
if len(manifestByPath) != len(watermarks) {
return fmt.Errorf("verify snapshot file/watermark count differs: manifest=%d database=%d", len(manifestByPath), len(watermarks))
}
for _, watermark := range watermarks {
path := filepath.Join("backup", watermark.RelativePath)
key := snapshotPathKey(path)
file, ok := manifestByPath[key]
if !ok {
return fmt.Errorf("verify snapshot manifest missing DB watermark: %s", filepath.ToSlash(path))
}
if file.Bytes != watermark.Bytes || file.IndexRecords != watermark.IndexRecords || file.AccountID != watermark.AccountID || file.Folder != watermark.Folder {
return fmt.Errorf("verify snapshot watermark mismatch: %s", file.Path)
}
}
return nil
}
func verifySnapshotReindex(root, databasePath string, expectedIndexRecords int64, expectedIndexSHA256 string) error {
temporaryRoot, err := os.MkdirTemp("", "mail-graveyard-snapshot-verify-")
if err != nil {
return fmt.Errorf("verify snapshot temporary restore: %w", err)
}
defer os.RemoveAll(temporaryRoot)
temporaryDB := filepath.Join(temporaryRoot, "mail-graveyard.db")
databaseInfo, err := os.Stat(databasePath)
if err != nil {
return fmt.Errorf("verify snapshot temporary database stat: %w", err)
}
if _, err := copyFilePrefix(databasePath, temporaryDB, databaseInfo.Size()); err != nil {
return fmt.Errorf("verify snapshot temporary database copy: %w", err)
}
oldCfg, oldDB := Cfg, DB
Cfg.DBPath = temporaryDB
Cfg.MboxRoot = filepath.Join(root, "backup")
DB = nil
defer func() {
if DB != nil {
_ = DB.Close()
}
Cfg, DB = oldCfg, oldDB
}()
if err := ConnectDB(false); err != nil {
return fmt.Errorf("verify snapshot restore database: %w", err)
}
if _, err := DB.Exec(`CREATE TABLE snapshot_verify_expected_index AS
SELECT account_id, folder, seq, message_id, body_sha256 FROM mbox_index`); err != nil {
return fmt.Errorf("verify snapshot preserve expected index: %w", err)
}
if err := ReindexArchives("all"); err != nil {
return fmt.Errorf("verify snapshot restore reindex: %w", err)
}
var reindexed int64
if err := DB.QueryRow(`SELECT count(*) FROM mbox_index`).Scan(&reindexed); err != nil {
return fmt.Errorf("verify snapshot restore index count: %w", err)
}
if reindexed != expectedIndexRecords {
return fmt.Errorf("verify snapshot restore index count=%d, want %d", reindexed, expectedIndexRecords)
}
reindexedDigest, err := indexDigest(DB)
if err != nil {
return err
}
if reindexedDigest != expectedIndexSHA256 {
difference, differenceErr := snapshotIndexDifference(DB)
if differenceErr != nil {
return fmt.Errorf("verify snapshot restore index digest=%s, want %s (difference query: %v)", reindexedDigest, expectedIndexSHA256, differenceErr)
}
return fmt.Errorf("verify snapshot restore index digest=%s, want %s (%s)", reindexedDigest, expectedIndexSHA256, difference)
}
stats, err := snapshotDatabaseStats(DB)
if err != nil {
return err
}
if stats.MissingArchiveRecords != 0 {
return fmt.Errorf("verify snapshot restore has %d copied identities without archive index", stats.MissingArchiveRecords)
}
if stats.InvalidStageOrder != 0 {
return fmt.Errorf("verify snapshot restore has %d precise identities with invalid stage order", stats.InvalidStageOrder)
}
return nil
}
func snapshotIndexDifference(db *sql.DB) (string, error) {
type differenceRow struct {
accountID int64
folder string
seq int64
messageID string
bodyHash string
}
queries := []struct {
label string
left string
right string
}{
{"missing after reindex", "snapshot_verify_expected_index", "mbox_index"},
{"unexpected after reindex", "mbox_index", "snapshot_verify_expected_index"},
}
parts := make([]string, 0, len(queries))
for _, query := range queries {
statement := fmt.Sprintf(`SELECT account_id, folder, seq, message_id, body_sha256 FROM %s
EXCEPT SELECT account_id, folder, seq, message_id, body_sha256 FROM %s`, query.left, query.right)
var count int64
if err := db.QueryRow(`SELECT count(*) FROM (` + statement + `)`).Scan(&count); err != nil {
return "", err
}
rows, err := db.Query(statement + ` LIMIT 3`)
if err != nil {
return "", err
}
var samples []string
for rows.Next() {
var row differenceRow
if err := rows.Scan(&row.accountID, &row.folder, &row.seq, &row.messageID, &row.bodyHash); err != nil {
_ = rows.Close()
return "", err
}
hash := row.bodyHash
if len(hash) > 16 {
hash = hash[:16]
}
samples = append(samples, fmt.Sprintf("account=%d folder=%q seq=%d id=%q hash=%s", row.accountID, row.folder, row.seq, row.messageID, hash))
}
if err := rows.Err(); err != nil {
_ = rows.Close()
return "", err
}
if err := rows.Close(); err != nil {
return "", err
}
parts = append(parts, fmt.Sprintf("%s=%d samples=[%s]", query.label, count, strings.Join(samples, "; ")))
}
return strings.Join(parts, ", "), nil
}
func pathInsideSnapshot(root, relative string) (string, error) {
if relative == "" || filepath.IsAbs(relative) {
return "", fmt.Errorf("verify snapshot invalid relative path: %q", relative)
}
path := filepath.Join(root, filepath.FromSlash(relative))
pathAbs, err := filepath.Abs(path)
if err != nil {
return "", err
}
rel, err := filepath.Rel(root, pathAbs)
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
return "", fmt.Errorf("verify snapshot path escapes root: %q", relative)
}
return pathAbs, nil
}
func hashFile(path string) (string, error) {
f, err := os.Open(path)
if err != nil {
return "", err
}
defer f.Close()
hash := sha256.New()
if _, err := io.Copy(hash, f); err != nil {
return "", err
}
return hex.EncodeToString(hash.Sum(nil)), nil
}
func syncFile(path string) error {
f, err := os.OpenFile(path, os.O_RDWR, 0)
if err != nil {
return err
}
defer f.Close()
return f.Sync()
}
func snapshotBytes(files []SnapshotFile) int64 {
var total int64
for _, file := range files {
total += file.Bytes
}
return total
}
func snapshotPathKey(path string) string {
path = filepath.Clean(path)
if runtime.GOOS == "windows" {
return strings.ToLower(path)
}
return path
}
func sortSnapshotFiles(files []SnapshotFile) {
sort.Slice(files, func(i, j int) bool { return files[i].Path < files[j].Path })
}

209
backend/14-snapshot_test.go Normal file
View file

@ -0,0 +1,209 @@
package backend
import (
"encoding/json"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
)
func TestSnapshotUsesDatabaseWatermarkWhileArchiveKeepsGrowing(t *testing.T) {
for _, compression := range []string{"none", "zstd"} {
t.Run(compression, func(t *testing.T) {
account, writer, restore := setupSnapshotTest(t, compression)
defer restore()
first := RawMessage{
MessageID: "first@example.com",
Body: []byte("Message-ID: <first@example.com>\r\nSubject: First\r\n\r\nFirst body\r\nFrom quoted\r\n"),
InternalDate: time.Unix(1_700_000_000, 0),
}
firstInfo := appendSnapshotTestMessage(t, account, writer, "INBOX", first)
snapshotRoot := filepath.Join(t.TempDir(), "snapshot")
var secondInfo MboxAppendInfo
err := createSnapshot(snapshotRoot, func() error {
second := RawMessage{
MessageID: "second@example.com",
Body: []byte("Message-ID: <second@example.com>\r\nSubject: Second\r\n\r\nSecond body"),
InternalDate: time.Unix(1_700_000_100, 0),
}
secondInfo = appendSnapshotTestMessage(t, account, writer, "INBOX", second)
return nil
})
if err != nil {
t.Fatal(err)
}
manifest := readSnapshotTestManifest(t, snapshotRoot)
if runtime.GOOS != "windows" {
databaseInfo, err := os.Stat(filepath.Join(snapshotRoot, filepath.FromSlash(manifest.DatabasePath)))
if err != nil {
t.Fatal(err)
}
if permissions := databaseInfo.Mode().Perm(); permissions != 0o600 {
t.Fatalf("snapshot database permissions=%#o, want 0600", permissions)
}
}
if manifest.IndexRecords != 1 || manifest.PreciseCopied != 1 || len(manifest.Files) != 1 {
t.Fatalf("snapshot crossed DB boundary: %#v", manifest)
}
if manifest.Files[0].Bytes != firstInfo.FileOffset+firstInfo.FrameLen {
t.Fatalf("snapshot bytes=%d, want first committed boundary=%d", manifest.Files[0].Bytes, firstInfo.FileOffset+firstInfo.FrameLen)
}
sourceInfo, err := os.Stat(secondInfo.Path)
if err != nil {
t.Fatal(err)
}
if sourceInfo.Size() <= manifest.Files[0].Bytes {
t.Fatalf("source did not grow beyond snapshot boundary: source=%d snapshot=%d", sourceInfo.Size(), manifest.Files[0].Bytes)
}
snapshotMbox := filepath.Join(snapshotRoot, filepath.FromSlash(manifest.Files[0].Path))
messages, err := readMboxMessages(snapshotMbox)
if err != nil {
t.Fatal(err)
}
if len(messages) != 1 || !strings.Contains(string(messages[0]), "First body") {
t.Fatalf("snapshot messages=%d, expected only first message", len(messages))
}
if err := VerifySnapshot(snapshotRoot); err != nil {
t.Fatalf("verify valid snapshot: %v", err)
}
var liveIndex int
if err := DB.QueryRow(`SELECT count(*) FROM mbox_index WHERE account_id=?`, account.ID).Scan(&liveIndex); err != nil {
t.Fatal(err)
}
if liveIndex != 2 {
t.Fatalf("snapshot verification altered live database: index=%d", liveIndex)
}
f, err := os.OpenFile(snapshotMbox, os.O_APPEND|os.O_WRONLY, 0)
if err != nil {
t.Fatal(err)
}
if _, err := f.Write([]byte("tamper")); err != nil {
_ = f.Close()
t.Fatal(err)
}
if err := f.Close(); err != nil {
t.Fatal(err)
}
if err := VerifySnapshot(snapshotRoot); err == nil {
t.Fatal("verification accepted a modified mbox")
}
})
}
}
func TestSnapshotRefusesKnownMissingArchiveIdentity(t *testing.T) {
account, _, restore := setupSnapshotTest(t, "none")
defer restore()
identity := MessageIdentity{MessageID: "missing@example.com", BodySHA256: bodySHA256([]byte("missing"))}
if err := MarkIdentityCopied(account.ID, "INBOX", identity); err != nil {
t.Fatal(err)
}
target := filepath.Join(t.TempDir(), "refused")
err := CreateSnapshot(target)
if err == nil || !strings.Contains(err.Error(), "no archive index") {
t.Fatalf("snapshot error=%v, want missing archive refusal", err)
}
if _, statErr := os.Stat(target); !os.IsNotExist(statErr) {
t.Fatalf("refused snapshot unexpectedly finalized: %v", statErr)
}
}
func TestSnapshotDoesNotOverwriteExistingTarget(t *testing.T) {
_, _, restore := setupSnapshotTest(t, "none")
defer restore()
target := filepath.Join(t.TempDir(), "existing")
if err := os.Mkdir(target, 0o700); err != nil {
t.Fatal(err)
}
sentinel := filepath.Join(target, "keep")
if err := os.WriteFile(sentinel, []byte("keep"), 0o600); err != nil {
t.Fatal(err)
}
if err := CreateSnapshot(target); err == nil {
t.Fatal("snapshot overwrote an existing target")
}
if got, err := os.ReadFile(sentinel); err != nil || string(got) != "keep" {
t.Fatalf("existing target changed: %q %v", got, err)
}
}
func setupSnapshotTest(t *testing.T, compression string) (Account, *MboxWriter, func()) {
t.Helper()
oldCfg, oldDB := Cfg, DB
root := t.TempDir()
Cfg = Config{
DBPath: filepath.Join(root, "mail-graveyard.db"),
MboxRoot: filepath.Join(root, "backup"),
MboxCompression: compression,
}
DB = nil
if err := ConnectDB(true); err != nil {
t.Fatal(err)
}
restore := func() {
if DB != nil {
_ = DB.Close()
}
Cfg, DB = oldCfg, oldDB
}
if err := SaveAccount(Account{
Name: "snapshot-account", SrcHost: "source.example", SrcPort: 993, SrcSecurity: "tls", SrcUser: "source@example.com", SrcPass: "x",
DstHost: "target.example", DstPort: 993, DstSecurity: "tls", DstUser: "target@example.com", DstPass: "x",
MboxDir: "snapshot@example.com", Active: true,
}); err != nil {
restore()
t.Fatal(err)
}
account, err := GetAccount("snapshot-account")
if err != nil {
restore()
t.Fatal(err)
}
writer, err := NewMboxWriter(filepath.Join(Cfg.MboxRoot, account.MboxDir))
if err != nil {
restore()
t.Fatal(err)
}
return account, writer, restore
}
func appendSnapshotTestMessage(t *testing.T, account Account, writer *MboxWriter, folder string, message RawMessage) MboxAppendInfo {
t.Helper()
identity := identityForRawMessage(message)
info, err := writer.Append(folder, message)
if err != nil {
t.Fatal(err)
}
if err := SaveMboxIndex(MboxIndexEntry{
AccountID: account.ID, Folder: safeMboxName(folder), MessageID: identity.MessageID, BodySHA256: identity.BodySHA256,
FileOffset: info.FileOffset, FrameLen: info.FrameLen, InnerOffset: info.InnerOffset, InnerLen: info.InnerLen,
}); err != nil {
t.Fatal(err)
}
if err := UpdateMboxIndexState(account.ID, safeMboxName(folder), info.FileOffset+info.FrameLen); err != nil {
t.Fatal(err)
}
if err := MarkIdentityCopied(account.ID, safeMboxName(folder), identity); err != nil {
t.Fatal(err)
}
return info
}
func readSnapshotTestManifest(t *testing.T, root string) SnapshotManifest {
t.Helper()
b, err := os.ReadFile(filepath.Join(root, "snapshot-manifest.json"))
if err != nil {
t.Fatal(err)
}
var manifest SnapshotManifest
if err := json.Unmarshal(b, &manifest); err != nil {
t.Fatal(err)
}
return manifest
}

16
main.go
View file

@ -25,16 +25,32 @@ func main() {
checkAccount := flag.String("check-account", "", "Konto-Logins pruefen und INBOX zaehlen, ohne zu kopieren")
reindex := flag.String("reindex", "", "Archiv-mbox-Index fuer Konto/Archiv oder 'all' neu aufbauen")
dedupTarget := flag.String("dedup-target", "", "Ziel-Postfach nach Message-ID-Dubletten scannen (Dry-Run ohne --apply)")
snapshot := flag.String("snapshot", "", "konsistenten DB- und mbox-Praefix-Snapshot in ein neues Zielverzeichnis schreiben")
verifySnapshot := flag.String("verify-snapshot", "", "Snapshot isoliert pruefen und dessen DB-Kopie vollstaendig reindexieren")
apply := flag.Bool("apply", false, "Scharfe Ausfuehrung fuer riskante Wartungsbefehle wie --dedup-target")
initDB := flag.Bool("init-db", false, "SQLite-DB neu anlegen, falls db_path noch nicht existiert")
flag.Parse()
if *snapshot != "" && (*verifySnapshot != "" || *runOnce != "" || *seedAccount != "" || *checkAccount != "" || *reindex != "" || *dedupTarget != "" || *watch) {
log.Fatal("-snapshot kann nicht mit einem anderen Betriebsmodus kombiniert werden")
}
if *verifySnapshot != "" && (*runOnce != "" || *seedAccount != "" || *checkAccount != "" || *reindex != "" || *dedupTarget != "" || *watch) {
log.Fatal("-verify-snapshot kann nicht mit einem anderen Betriebsmodus kombiniert werden")
}
if err := backend.LoadConfig("config.json"); err != nil {
log.Fatalf("config: %v", err)
}
if *verifySnapshot != "" {
must(backend.VerifySnapshot(*verifySnapshot))
return
}
if err := backend.ConnectDB(*initDB); err != nil {
log.Fatalf("database: %v", err)
}
if *snapshot != "" {
must(backend.CreateSnapshot(*snapshot))
return
}
if err := backend.InitAuth(); err != nil {
log.Fatalf("auth: %v", err)
}