Add verified snapshots and harden archive identity
This commit is contained in:
DonVoo
parent
c6de214ac8
commit
68b9cee880
9 changed files with 1372 additions and 52 deletions
34
INSTALL.md
34
INSTALL.md
|
|
@ -47,8 +47,10 @@ Ohne Web, z. B. als Cronjob oder Dauerlauf im Umzugsfenster:
|
|||
./mail-graveyard --run dr-gold --watch # ein Konto, Delta-Schleife bis Stop
|
||||
```
|
||||
|
||||
Dank Message-ID-Cache kopiert jeder Lauf nur Neues — beliebig oft wiederholbar
|
||||
ohne Dubletten. Für den Dauerlauf reicht `nohup`/systemd/Aufgabenplanung.
|
||||
Die kanonische Inhaltsidentitaet verhindert dabei doppelte **Schreibvorgaenge**
|
||||
in Archiv und Ziel. Der aktuelle Volllauf laedt trotzdem jede Quellmail erneut,
|
||||
bevor er deren Body-Hash pruefen kann. Grosse Konten deshalb bis zum
|
||||
UID-Delta-Watcher nicht in kurzen Intervallen als Dauerlauf starten.
|
||||
|
||||
## 5. Wartung
|
||||
|
||||
|
|
@ -56,18 +58,46 @@ ohne Dubletten. Für den Dauerlauf reicht `nohup`/systemd/Aufgabenplanung.
|
|||
./mail-graveyard --reindex all
|
||||
./mail-graveyard --dedup-target konto-name # Dry-Run
|
||||
./mail-graveyard --dedup-target konto-name --apply # loescht Dubletten im Ziel
|
||||
./mail-graveyard --snapshot /sicherungen/stand-2026-07-16
|
||||
./mail-graveyard --verify-snapshot /sicherungen/stand-2026-07-16
|
||||
```
|
||||
|
||||
`--reindex` baut den Archiv-Index fuer vorhandene mbox/mbox.zst-Dateien neu auf.
|
||||
`--dedup-target` ist bewusst ein Dry-Run, bis `--apply` gesetzt wird; Mails ohne
|
||||
Message-ID werden nie geloescht.
|
||||
|
||||
`--snapshot` erzeugt zuerst per SQLite `VACUUM INTO` einen konsistenten
|
||||
DB-Stand. Anschliessend kopiert es aus jeder append-only mbox exakt das durch
|
||||
`MAX(file_offset+frame_len)` **dieses DB-Standes** belegte Praefix. Das Ziel
|
||||
muss neu sein und enthaelt `data/mail-graveyard.db`, `backup/...` sowie ein
|
||||
Manifest mit Grenzen und SHA-256-Pruefsummen. Ein Fehler hinterlaesst ein klar
|
||||
benanntes `.partial-*`-Verzeichnis und wird nie als fertiger Stand umbenannt.
|
||||
Die Datenbank wird wegen der enthaltenen Klartext-Zugangsdaten mit `0600`
|
||||
geschrieben; die Verifikation weist unter Linux weiter gefasste Rechte zurueck.
|
||||
|
||||
`--verify-snapshot` veraendert weder Produktion noch Snapshot: Es prueft
|
||||
Manifest, Dateigroessen, SHA-256, SQLite-Integritaet, Stufenreihenfolge und den
|
||||
Identitaetsdigest. Danach kopiert es nur die Snapshot-DB in ein temporaeres
|
||||
Restore-Verzeichnis und reindiziert dort alle mbox-Dateien. Der
|
||||
Identitaetsdigest muss vor und nach dem Reindex gleich bleiben.
|
||||
|
||||
Der lokale Snapshot ist nur die konsistente **Vorstufe** eines Backups. Er
|
||||
enthaelt Kontozugangsdaten und muss anschliessend verschluesselt auf einen
|
||||
raeumlich getrennten Datentraeger uebertragen werden.
|
||||
|
||||
## 6. Sicherheit
|
||||
|
||||
Tool hält fremde IMAP-Passwörter und kann Mails senden → Default bind
|
||||
`127.0.0.1`. Falls remote nötig: hinter Caddy + `/vadmin`-Mail-2FA. `config.json`
|
||||
und `*.db` nie committen.
|
||||
|
||||
Die Live-SQLite-Datei sowie ihre WAL-/SHM-Sidecars werden bei jedem Start auf
|
||||
`0600` gesetzt. Das gilt auch fuer weitere `*.db`, `*.db-wal` und `*.db-shm`
|
||||
direkt im konfigurierten Datenverzeichnis sowie fuer die Datenbank eines neu
|
||||
erzeugten Snapshots. Kann ein Dateisystem Unix-Rechte nicht abbilden (z. B.
|
||||
bestimmte exFAT-/CIFS-Mounts), startet das Werkzeug bewusst nicht: Zugangsdaten
|
||||
bleiben fail-closed statt unbemerkt lesbar.
|
||||
|
||||
## 7. Reihenfolge bei einem echten Umzug
|
||||
|
||||
1. Ziel-Postfächer beim neuen Hoster anlegen (`archiv-…`, `temp-…`).
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ import (
|
|||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
|
|
@ -85,6 +87,10 @@ func ConnectDB(initDB bool) error {
|
|||
_ = db.Close()
|
||||
return err
|
||||
}
|
||||
if err := secureSQLiteFiles(Cfg.DBPath); err != nil {
|
||||
_ = db.Close()
|
||||
return err
|
||||
}
|
||||
schema := []string{
|
||||
`CREATE TABLE IF NOT EXISTS accounts(
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
|
|
@ -361,7 +367,7 @@ func ensureDBFile(path string, initDB bool) error {
|
|||
if info.IsDir() {
|
||||
return fmt.Errorf("DB %q ist ein Verzeichnis", path)
|
||||
}
|
||||
return nil
|
||||
return chmodPrivateFile(path)
|
||||
}
|
||||
if !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
|
|
@ -373,7 +379,57 @@ func ensureDBFile(path string, initDB bool) error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return f.Close()
|
||||
if err := f.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return chmodPrivateFile(path)
|
||||
}
|
||||
|
||||
func secureSQLiteFiles(path string) error {
|
||||
directory := filepath.Dir(path)
|
||||
entries, err := os.ReadDir(directory)
|
||||
if err != nil {
|
||||
return fmt.Errorf("scan SQLite directory %q: %w", directory, err)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if entry.Type()&os.ModeSymlink != 0 || !isSQLiteDataFile(entry.Name()) {
|
||||
continue
|
||||
}
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return fmt.Errorf("inspect SQLite file %q: %w", filepath.Join(directory, entry.Name()), err)
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
continue
|
||||
}
|
||||
candidate := filepath.Join(directory, entry.Name())
|
||||
if err := chmodPrivateFile(candidate); err != nil {
|
||||
return fmt.Errorf("secure SQLite file %q: %w", candidate, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func chmodPrivateFile(path string) error {
|
||||
if err := os.Chmod(path, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
if runtime.GOOS == "windows" {
|
||||
return nil
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if permissions := info.Mode().Perm(); permissions&0o077 != 0 {
|
||||
return fmt.Errorf("permissions=%#o after chmod 0600", permissions)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isSQLiteDataFile(name string) bool {
|
||||
name = strings.ToLower(name)
|
||||
return strings.HasSuffix(name, ".db") || strings.HasSuffix(name, ".db-wal") || strings.HasSuffix(name, ".db-shm")
|
||||
}
|
||||
|
||||
func ensureAppUserColumns() error {
|
||||
|
|
@ -724,56 +780,72 @@ func GetCopyIdentityStateWithFolderAlias(accountID int64, folder, legacyFolder s
|
|||
if legacyFolder != "" && legacyFolder != folder {
|
||||
folders = append(folders, legacyFolder)
|
||||
}
|
||||
aliases := []string{identity.MessageID, identity.LegacyMessageID}
|
||||
aliases = append(aliases, identity.LegacyMessageIDs...)
|
||||
if identity.MessageID == "" && identity.BodySHA256 != "" {
|
||||
aliases = append(aliases, "sha256:"+identity.BodySHA256)
|
||||
}
|
||||
uniqueAliases := make([]string, 0, len(aliases))
|
||||
seenAliases := map[string]bool{}
|
||||
for _, alias := range aliases {
|
||||
alias = strings.TrimSpace(alias)
|
||||
if alias == "" || seenAliases[alias] {
|
||||
continue
|
||||
}
|
||||
seenAliases[alias] = true
|
||||
uniqueAliases = append(uniqueAliases, alias)
|
||||
}
|
||||
// Precise rows always win. For Message-ID-less mail the hash is the whole
|
||||
// identity, irrespective of which legacy key happened to be stored beside it.
|
||||
// For mail with an ID, historical normalizations are precise aliases only
|
||||
// when the canonical body hash is identical.
|
||||
var mboxDone, targetDone int
|
||||
for _, candidateFolder := range folders {
|
||||
var mbox, target int
|
||||
var err error
|
||||
if identity.MessageID == "" {
|
||||
err = DB.QueryRow(`SELECT COALESCE(MAX(mbox_done),0), COALESCE(MAX(target_done),0) FROM copied
|
||||
var mbox, target int
|
||||
err := DB.QueryRow(`SELECT COALESCE(MAX(mbox_done),0), COALESCE(MAX(target_done),0) FROM copied
|
||||
WHERE account_id=? AND folder=? AND body_sha256=?`, accountID, candidateFolder, identity.BodySHA256).Scan(&mbox, &target)
|
||||
} else {
|
||||
err = DB.QueryRow(`SELECT COALESCE(MAX(mbox_done),0), COALESCE(MAX(target_done),0) FROM copied
|
||||
WHERE account_id=? AND folder=? AND message_id=? AND body_sha256=?`,
|
||||
accountID, candidateFolder, identity.MessageID, identity.BodySHA256).Scan(&mbox, &target)
|
||||
if err != nil {
|
||||
return CopyState{}, err
|
||||
}
|
||||
mboxDone |= mbox
|
||||
targetDone |= target
|
||||
continue
|
||||
}
|
||||
for _, alias := range uniqueAliases {
|
||||
var mbox, target int
|
||||
err := DB.QueryRow(`SELECT COALESCE(MAX(mbox_done),0), COALESCE(MAX(target_done),0) FROM copied
|
||||
WHERE account_id=? AND folder=? AND message_id=? AND body_sha256=?`,
|
||||
accountID, candidateFolder, alias, identity.BodySHA256).Scan(&mbox, &target)
|
||||
if err != nil {
|
||||
return CopyState{}, err
|
||||
}
|
||||
mboxDone |= mbox
|
||||
targetDone |= target
|
||||
}
|
||||
}
|
||||
if mboxDone != 0 || targetDone != 0 {
|
||||
return CopyState{MboxDone: mboxDone != 0, TargetDone: targetDone != 0}, nil
|
||||
}
|
||||
// Once a Message-ID has precise archive identities, a different body with
|
||||
// that same ID is new and must not be hidden by the old wildcard row.
|
||||
if identity.MessageID != "" && identity.BodySHA256 != "" {
|
||||
if len(uniqueAliases) != 0 && identity.BodySHA256 != "" {
|
||||
var precise int
|
||||
for _, candidateFolder := range folders {
|
||||
for _, alias := range uniqueAliases {
|
||||
var count int
|
||||
if err := DB.QueryRow(`SELECT count(*) FROM copied WHERE account_id=? AND folder=? AND message_id=? AND body_sha256<>''`,
|
||||
accountID, candidateFolder, identity.MessageID).Scan(&count); err != nil {
|
||||
accountID, candidateFolder, alias).Scan(&count); err != nil {
|
||||
return CopyState{}, err
|
||||
}
|
||||
precise += count
|
||||
}
|
||||
}
|
||||
if precise > 0 {
|
||||
return CopyState{}, nil
|
||||
}
|
||||
}
|
||||
aliases := []string{identity.MessageID, identity.LegacyMessageID}
|
||||
if identity.MessageID == "" && identity.BodySHA256 != "" {
|
||||
aliases = append(aliases, "sha256:"+identity.BodySHA256)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, alias := range aliases {
|
||||
alias = strings.TrimSpace(alias)
|
||||
if alias == "" || seen[alias] {
|
||||
continue
|
||||
}
|
||||
seen[alias] = true
|
||||
for _, alias := range uniqueAliases {
|
||||
for _, candidateFolder := range folders {
|
||||
var mbox, target int
|
||||
err := DB.QueryRow(`SELECT COALESCE(MAX(mbox_done),0), COALESCE(MAX(target_done),0) FROM copied
|
||||
|
|
@ -875,21 +947,25 @@ func ReplaceMboxIndex(accountID int64, folder string, entries []MboxIndexEntry,
|
|||
if err := aliasRows.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
legacyIDsBySeq := map[int]string{}
|
||||
type storedIndexIdentity struct {
|
||||
messageID string
|
||||
bodySHA256 string
|
||||
}
|
||||
storedIdentityBySeq := map[int]storedIndexIdentity{}
|
||||
for _, candidateFolder := range folderAliases {
|
||||
rows, err := tx.Query(`SELECT seq, message_id FROM mbox_index WHERE account_id=? AND folder=?`, accountID, candidateFolder)
|
||||
rows, err := tx.Query(`SELECT seq, message_id, body_sha256 FROM mbox_index WHERE account_id=? AND folder=?`, accountID, candidateFolder)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for rows.Next() {
|
||||
var seq int
|
||||
var messageID string
|
||||
if err := rows.Scan(&seq, &messageID); err != nil {
|
||||
var identity storedIndexIdentity
|
||||
if err := rows.Scan(&seq, &identity.messageID, &identity.bodySHA256); err != nil {
|
||||
_ = rows.Close()
|
||||
return err
|
||||
}
|
||||
if _, exists := legacyIDsBySeq[seq]; !exists || candidateFolder == folder {
|
||||
legacyIDsBySeq[seq] = messageID
|
||||
if _, exists := storedIdentityBySeq[seq]; !exists || candidateFolder == folder {
|
||||
storedIdentityBySeq[seq] = identity
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
|
|
@ -907,6 +983,15 @@ func ReplaceMboxIndex(accountID int64, folder string, entries []MboxIndexEntry,
|
|||
entry.AccountID = accountID
|
||||
entry.Folder = folder
|
||||
entry.Seq = i
|
||||
storedIdentity := storedIdentityBySeq[i]
|
||||
// The IMAP envelope and a later RFC-822 header parse can represent the
|
||||
// same malformed Message-ID differently. If the canonical body at the
|
||||
// same archive position is identical, retain the already committed ID so
|
||||
// a rebuild is an exact metadata restore. Never carry it across a body
|
||||
// change, because that could attach an old identity to different content.
|
||||
if storedIdentity.bodySHA256 != "" && storedIdentity.bodySHA256 == entry.BodySHA256 {
|
||||
entry.MessageID = storedIdentity.messageID
|
||||
}
|
||||
if entry.MessageID == "" && entry.BodySHA256 == "" {
|
||||
continue
|
||||
}
|
||||
|
|
@ -921,7 +1006,7 @@ func ReplaceMboxIndex(accountID int64, folder string, entries []MboxIndexEntry,
|
|||
// pipeline the target append happened before the mbox append, so recovered
|
||||
// records are complete in both stages. Marking them prevents a later watch
|
||||
// run from appending duplicates to either destination.
|
||||
state, found, err := copyStateForReindex(tx, entry.AccountID, folderAliases, entry, legacyIDsBySeq[i])
|
||||
state, found, err := copyStateForReindex(tx, entry.AccountID, folderAliases, entry, storedIdentity.messageID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,10 +31,13 @@ type RawMessage struct {
|
|||
// for byte-different messages. Messages without a Message-ID are identified by
|
||||
// BodySHA256 alone. LegacyMessageID keeps the old raw-IMAP hash addressable
|
||||
// while existing databases are migrated without re-copying mail.
|
||||
// LegacyMessageIDs additionally retain historical Message-ID normalizations
|
||||
// from both the IMAP envelope and the raw RFC-822 header.
|
||||
type MessageIdentity struct {
|
||||
MessageID string
|
||||
BodySHA256 string
|
||||
LegacyMessageID string
|
||||
LegacyMessageIDs []string
|
||||
}
|
||||
|
||||
type MessageHeader struct {
|
||||
|
|
@ -489,19 +492,23 @@ func messageID(body []byte) string {
|
|||
|
||||
func identityForRawMessage(m RawMessage) MessageIdentity {
|
||||
id := normalizeMessageID(m.MessageID)
|
||||
legacyID := id
|
||||
legacyID := legacyNormalizeMessageID(m.MessageID)
|
||||
aliases := []string{legacyID}
|
||||
if strings.HasPrefix(strings.ToLower(id), "sha256:") {
|
||||
id = ""
|
||||
}
|
||||
if id == "" {
|
||||
if msg, err := mail.ReadMessage(bytes.NewReader(m.Body)); err == nil {
|
||||
id = normalizeMessageID(msg.Header.Get("Message-ID"))
|
||||
headerID := msg.Header.Get("Message-ID")
|
||||
aliases = append(aliases, normalizeMessageID(headerID), legacyNormalizeMessageID(headerID))
|
||||
if id == "" {
|
||||
id = normalizeMessageID(headerID)
|
||||
}
|
||||
}
|
||||
return MessageIdentity{
|
||||
MessageID: id,
|
||||
BodySHA256: bodySHA256(m.Body),
|
||||
LegacyMessageID: legacyID,
|
||||
LegacyMessageIDs: aliases,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -525,6 +532,18 @@ func bodySHA256(raw []byte) string {
|
|||
}
|
||||
|
||||
func normalizeMessageID(id string) string {
|
||||
id = strings.TrimSpace(id)
|
||||
if start := strings.IndexByte(id, '<'); start >= 0 {
|
||||
if relativeEnd := strings.IndexByte(id[start+1:], '>'); relativeEnd >= 0 {
|
||||
if candidate := strings.TrimSpace(id[start+1 : start+1+relativeEnd]); candidate != "" {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
}
|
||||
return strings.Trim(id, "<>")
|
||||
}
|
||||
|
||||
func legacyNormalizeMessageID(id string) string {
|
||||
return strings.Trim(strings.TrimSpace(id), "<>")
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -25,6 +25,56 @@ func TestMessageIDFallsBackToBodyHash(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestNormalizeMessageIDExtractsBracketedIDBeforeBrokenComment(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"<normal@example.com>": "normal@example.com",
|
||||
"bracketless@example.com": "bracketless@example.com",
|
||||
" <5486CB7400AF1FB7@mr001msb.fastweb.it> (added by postmaster) ": "5486CB7400AF1FB7@mr001msb.fastweb.it",
|
||||
"comment <embedded@example.com> trailing": "embedded@example.com",
|
||||
}
|
||||
for input, want := range tests {
|
||||
if got := normalizeMessageID(input); got != want {
|
||||
t.Errorf("normalizeMessageID(%q)=%q, want %q", input, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIdentityKeepsOldMalformedHeaderNormalizationAsAlias(t *testing.T) {
|
||||
tests := []struct {
|
||||
header string
|
||||
clean string
|
||||
legacy string
|
||||
}{
|
||||
{
|
||||
header: `<476BDFDB019289BE@mail21.bluewin.ch> (added by postmaster@bluewin.ch)`,
|
||||
clean: `476BDFDB019289BE@mail21.bluewin.ch`,
|
||||
legacy: `476BDFDB019289BE@mail21.bluewin.ch> (added by postmaster@bluewin.ch)`,
|
||||
},
|
||||
{
|
||||
header: `<22578196653698419217.BB8A0E8F48F3F290@dr-gold.de>+D271B1409C3C6028`,
|
||||
clean: `22578196653698419217.BB8A0E8F48F3F290@dr-gold.de`,
|
||||
legacy: `22578196653698419217.BB8A0E8F48F3F290@dr-gold.de>+D271B1409C3C6028`,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
body := []byte("Message-ID: " + tt.header + "\r\nSubject: malformed ID\r\n\r\nbody")
|
||||
identity := identityForRawMessage(RawMessage{MessageID: messageID(body), Body: body})
|
||||
if identity.MessageID != tt.clean {
|
||||
t.Fatalf("MessageID=%q, want %q", identity.MessageID, tt.clean)
|
||||
}
|
||||
found := false
|
||||
for _, alias := range identity.LegacyMessageIDs {
|
||||
if alias == tt.legacy {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("legacy alias %q missing from %#v", tt.legacy, identity.LegacyMessageIDs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBodySHA256IsStableAcrossMboxNormalization(t *testing.T) {
|
||||
raw := []byte("From: a@example.com\r\nSubject: Test\r\n\r\nFirst\r\nFrom escaped\r\n\r\n")
|
||||
record := mboxRecord(RawMessage{Body: raw})
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package backend
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
|
|
@ -290,17 +291,55 @@ func reindexAccountArchive(account Account) error {
|
|||
}
|
||||
|
||||
func reindexPlainMbox(path string) ([]MboxIndexEntry, error) {
|
||||
b, err := os.ReadFile(path)
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
reader := bufio.NewReaderSize(f, 256*1024)
|
||||
var out []MboxIndexEntry
|
||||
for _, part := range splitMboxRecordsWithOffsets(b) {
|
||||
entry := indexEntryFromMboxRecord(part.Message, part.Offset, int64(part.Length), 0, int64(part.Length))
|
||||
var record bytes.Buffer
|
||||
var offset int64
|
||||
var messageStart int64 = -1
|
||||
appendRecord := func(end int64) {
|
||||
if messageStart < 0 || end <= messageStart {
|
||||
return
|
||||
}
|
||||
message := unescapeMboxMessage(record.Bytes())
|
||||
if len(message) == 0 {
|
||||
return
|
||||
}
|
||||
entry := indexEntryFromMboxRecord(message, messageStart, end-messageStart, 0, end-messageStart)
|
||||
if entry.MessageID != "" || entry.BodySHA256 != "" {
|
||||
out = append(out, entry)
|
||||
}
|
||||
}
|
||||
for {
|
||||
lineStart := offset
|
||||
line, readErr := reader.ReadBytes('\n')
|
||||
offset += int64(len(line))
|
||||
content := line
|
||||
if len(content) > 0 && content[len(content)-1] == '\n' {
|
||||
content = content[:len(content)-1]
|
||||
}
|
||||
if len(content) > 0 && content[len(content)-1] == '\r' {
|
||||
content = content[:len(content)-1]
|
||||
}
|
||||
if bytes.HasPrefix(content, []byte("From ")) {
|
||||
appendRecord(lineStart)
|
||||
record = bytes.Buffer{}
|
||||
messageStart = offset
|
||||
} else if messageStart >= 0 && len(line) > 0 {
|
||||
_, _ = record.Write(line)
|
||||
}
|
||||
if readErr != nil {
|
||||
if readErr != io.EOF {
|
||||
return nil, readErr
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
appendRecord(offset)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,8 +3,10 @@ package backend
|
|||
import (
|
||||
"bytes"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
|
@ -12,6 +14,39 @@ import (
|
|||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
func TestReindexPlainMboxStreamingMatchesReferenceParser(t *testing.T) {
|
||||
var mbox bytes.Buffer
|
||||
for i := 0; i < 257; i++ {
|
||||
body := strings.Repeat(fmt.Sprintf("line-%03d From inside body\r\n", i), i%31+1)
|
||||
mbox.Write(mboxRecord(RawMessage{
|
||||
MessageID: fmt.Sprintf("stream-%03d@example.com", i),
|
||||
Body: []byte(fmt.Sprintf(
|
||||
"Message-ID: <stream-%03d@example.com>\r\nSubject: Stream %03d\r\n\r\n%s",
|
||||
i, i, body,
|
||||
)),
|
||||
}))
|
||||
}
|
||||
path := filepath.Join(t.TempDir(), "streaming.mbox")
|
||||
if err := os.WriteFile(path, mbox.Bytes(), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := reindexPlainMbox(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
parts := splitMboxRecordsWithOffsets(mbox.Bytes())
|
||||
if len(got) != len(parts) {
|
||||
t.Fatalf("streaming entries=%d, reference parts=%d", len(got), len(parts))
|
||||
}
|
||||
for i, part := range parts {
|
||||
want := indexEntryFromMboxRecord(part.Message, part.Offset, int64(part.Length), 0, int64(part.Length))
|
||||
if got[i] != want {
|
||||
t.Fatalf("entry %d differs:\n got %#v\n want %#v", i, got[i], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestZstdMboxRoundTripAndIndexRead(t *testing.T) {
|
||||
oldCfg, oldDB := Cfg, DB
|
||||
root := t.TempDir()
|
||||
|
|
@ -325,6 +360,14 @@ func TestConnectDBConfiguresBusyTimeoutAndCopyStages(t *testing.T) {
|
|||
Cfg, DB = oldCfg, oldDB
|
||||
})
|
||||
Cfg = Config{DBPath: filepath.Join(root, "mail-graveyard.db"), MboxRoot: filepath.Join(root, "backup")}
|
||||
backupDB := filepath.Join(root, "PRE-existing-copy.db")
|
||||
if err := os.WriteFile(backupDB, []byte("diagnostic copy"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
nonDB := filepath.Join(root, "keep-mode.txt")
|
||||
if err := os.WriteFile(nonDB, []byte("not a database"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ConnectDB(true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -335,6 +378,24 @@ func TestConnectDBConfiguresBusyTimeoutAndCopyStages(t *testing.T) {
|
|||
if timeout != 10000 {
|
||||
t.Fatalf("busy_timeout=%d, want 10000", timeout)
|
||||
}
|
||||
if runtime.GOOS != "windows" {
|
||||
for _, path := range []string{Cfg.DBPath, Cfg.DBPath + "-wal", Cfg.DBPath + "-shm", backupDB} {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if permissions := info.Mode().Perm(); permissions != 0o600 {
|
||||
t.Fatalf("SQLite file %s permissions=%#o, want 0600", path, permissions)
|
||||
}
|
||||
}
|
||||
info, err := os.Stat(nonDB)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if permissions := info.Mode().Perm(); permissions != 0o644 {
|
||||
t.Fatalf("non-DB file permissions changed to %#o", permissions)
|
||||
}
|
||||
}
|
||||
if err := SaveAccount(Account{
|
||||
Name: "stage-account", SrcHost: "source.example", SrcPort: 993, SrcSecurity: "tls", SrcUser: "source@example.com", SrcPass: "x",
|
||||
DstHost: "target.example", DstPort: 993, DstSecurity: "tls", DstUser: "target@example.com", DstPass: "x", Active: true,
|
||||
|
|
@ -379,6 +440,106 @@ func TestConnectDBConfiguresBusyTimeoutAndCopyStages(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestReplaceMboxIndexPreservesStoredMessageIDOnlyForSameBody(t *testing.T) {
|
||||
oldCfg, oldDB := Cfg, DB
|
||||
root := t.TempDir()
|
||||
t.Cleanup(func() {
|
||||
if DB != nil {
|
||||
_ = DB.Close()
|
||||
}
|
||||
Cfg, DB = oldCfg, oldDB
|
||||
})
|
||||
Cfg = Config{DBPath: filepath.Join(root, "mail-graveyard.db"), MboxRoot: filepath.Join(root, "backup")}
|
||||
if err := ConnectDB(true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SaveAccount(Account{
|
||||
Name: "identity-restore", SrcHost: "source.example", SrcPort: 993, SrcSecurity: "tls", SrcUser: "source@example.com", SrcPass: "x",
|
||||
DstHost: "target.example", DstPort: 993, DstSecurity: "tls", DstUser: "target@example.com", DstPass: "x", Active: true,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
account, err := GetAccount("identity-restore")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stableHash := bodySHA256([]byte("same archived body"))
|
||||
if err := ReplaceMboxIndex(account.ID, "INBOX", []MboxIndexEntry{{
|
||||
MessageID: "stored@example.com> (added by postmaster)", BodySHA256: stableHash, FrameLen: 10, InnerLen: 10,
|
||||
}}, 10); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ReplaceMboxIndex(account.ID, "INBOX", []MboxIndexEntry{{
|
||||
MessageID: "parsed@example.com", BodySHA256: stableHash, FrameLen: 10, InnerLen: 10,
|
||||
}}, 10); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entry, err := GetMboxIndex(account.ID, "INBOX", 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if entry.MessageID != "stored@example.com> (added by postmaster)" {
|
||||
t.Fatalf("same body lost stored Message-ID: %q", entry.MessageID)
|
||||
}
|
||||
|
||||
changedHash := bodySHA256([]byte("different archived body"))
|
||||
if err := ReplaceMboxIndex(account.ID, "INBOX", []MboxIndexEntry{{
|
||||
MessageID: "new-content@example.com", BodySHA256: changedHash, FrameLen: 11, InnerLen: 11,
|
||||
}}, 11); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entry, err = GetMboxIndex(account.ID, "INBOX", 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if entry.MessageID != "new-content@example.com" {
|
||||
t.Fatalf("different body inherited stale Message-ID: %q", entry.MessageID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreciseLegacyMessageIDAliasesPreventLiveRecopy(t *testing.T) {
|
||||
account, _, restore := setupSnapshotTest(t, "none")
|
||||
defer restore()
|
||||
tests := []struct {
|
||||
header string
|
||||
legacy string
|
||||
}{
|
||||
{
|
||||
header: `<476BDFDB019289BE@mail21.bluewin.ch> (added by postmaster@bluewin.ch)`,
|
||||
legacy: `476BDFDB019289BE@mail21.bluewin.ch> (added by postmaster@bluewin.ch)`,
|
||||
},
|
||||
{
|
||||
header: `<22578196653698419217.BB8A0E8F48F3F290@dr-gold.de>+D271B1409C3C6028`,
|
||||
legacy: `22578196653698419217.BB8A0E8F48F3F290@dr-gold.de>+D271B1409C3C6028`,
|
||||
},
|
||||
}
|
||||
for i, tt := range tests {
|
||||
folder := fmt.Sprintf("INBOX.%d", i)
|
||||
body := []byte("Message-ID: " + tt.header + "\r\nSubject: malformed ID\r\n\r\noriginal body")
|
||||
identity := identityForRawMessage(RawMessage{MessageID: messageID(body), Body: body})
|
||||
if err := MarkIdentityCopied(account.ID, folder, MessageIdentity{MessageID: tt.legacy, BodySHA256: identity.BodySHA256}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state, err := GetCopyIdentityState(account.ID, folder, identity)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !state.MboxDone || !state.TargetDone {
|
||||
t.Fatalf("fixture %d precise legacy alias was not recognized: %#v", i, state)
|
||||
}
|
||||
|
||||
changedBody := []byte("Message-ID: " + tt.header + "\r\nSubject: malformed ID\r\n\r\ndifferent body")
|
||||
changedIdentity := identityForRawMessage(RawMessage{MessageID: messageID(changedBody), Body: changedBody})
|
||||
state, err = GetCopyIdentityState(account.ID, folder, changedIdentity)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state.MboxDone || state.TargetDone {
|
||||
t.Fatalf("fixture %d legacy alias hid byte-different mail: %#v", i, state)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIdentitySchemaUpgradePreservesLegacyAliases(t *testing.T) {
|
||||
oldCfg, oldDB := Cfg, DB
|
||||
root := t.TempDir()
|
||||
|
|
|
|||
711
backend/14-snapshot.go
Normal file
711
backend/14-snapshot.go
Normal file
|
|
@ -0,0 +1,711 @@
|
|||
package backend
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const snapshotFormatVersion = 1
|
||||
|
||||
type SnapshotManifest struct {
|
||||
FormatVersion int `json:"format_version"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
DatabasePath string `json:"database_path"`
|
||||
DatabaseSHA256 string `json:"database_sha256"`
|
||||
IndexSHA256 string `json:"index_sha256"`
|
||||
IndexRecords int64 `json:"index_records"`
|
||||
PreciseCopied int64 `json:"precise_copied"`
|
||||
PendingMbox int64 `json:"pending_mbox"`
|
||||
PendingTarget int64 `json:"pending_target"`
|
||||
InvalidStageOrder int64 `json:"invalid_stage_order"`
|
||||
MissingArchiveRecords int64 `json:"missing_archive_records"`
|
||||
Files []SnapshotFile `json:"files"`
|
||||
}
|
||||
|
||||
type SnapshotFile struct {
|
||||
Path string `json:"path"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
AccountID int64 `json:"account_id"`
|
||||
Account string `json:"account"`
|
||||
Folder string `json:"folder"`
|
||||
IndexRecords int64 `json:"index_records"`
|
||||
}
|
||||
|
||||
type snapshotWatermark struct {
|
||||
AccountID int64
|
||||
Account string
|
||||
MboxDir string
|
||||
Folder string
|
||||
Bytes int64
|
||||
IndexRecords int64
|
||||
SourcePath string
|
||||
RelativePath string
|
||||
}
|
||||
|
||||
// CreateSnapshot creates a self-consistent DB + mbox-prefix snapshot without
|
||||
// stopping append-only archive writers. The DB snapshot is taken first; every
|
||||
// copied mbox prefix is then bounded exclusively by offsets from that snapshot.
|
||||
func CreateSnapshot(target string) error {
|
||||
return createSnapshot(target, nil)
|
||||
}
|
||||
|
||||
func createSnapshot(target string, afterDatabaseSnapshot func() error) (retErr error) {
|
||||
if DB == nil {
|
||||
return fmt.Errorf("snapshot: database is not connected")
|
||||
}
|
||||
target = strings.TrimSpace(target)
|
||||
if target == "" {
|
||||
return fmt.Errorf("snapshot: target directory is empty")
|
||||
}
|
||||
targetAbs, err := filepath.Abs(target)
|
||||
if err != nil {
|
||||
return fmt.Errorf("snapshot target: %w", err)
|
||||
}
|
||||
if _, err := os.Stat(targetAbs); err == nil {
|
||||
return fmt.Errorf("snapshot target already exists: %s", targetAbs)
|
||||
} else if !os.IsNotExist(err) {
|
||||
return fmt.Errorf("snapshot target stat: %w", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(targetAbs), 0o700); err != nil {
|
||||
return fmt.Errorf("snapshot target parent: %w", err)
|
||||
}
|
||||
partial := fmt.Sprintf("%s.partial-%d-%d", targetAbs, os.Getpid(), time.Now().UnixNano())
|
||||
if err := os.Mkdir(partial, 0o700); err != nil {
|
||||
return fmt.Errorf("snapshot staging directory: %w", err)
|
||||
}
|
||||
complete := false
|
||||
defer func() {
|
||||
if !complete {
|
||||
log.Printf("snapshot incomplete; staging data retained at %s", partial)
|
||||
if retErr != nil {
|
||||
retErr = fmt.Errorf("%w (incomplete snapshot retained at %s)", retErr, partial)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
databaseRel := filepath.ToSlash(filepath.Join("data", "mail-graveyard.db"))
|
||||
databasePath := filepath.Join(partial, filepath.FromSlash(databaseRel))
|
||||
if err := os.MkdirAll(filepath.Dir(databasePath), 0o700); err != nil {
|
||||
return fmt.Errorf("snapshot database directory: %w", err)
|
||||
}
|
||||
if _, err := DB.Exec(`VACUUM INTO ?`, databasePath); err != nil {
|
||||
return fmt.Errorf("snapshot database VACUUM INTO: %w", err)
|
||||
}
|
||||
if err := chmodPrivateFile(databasePath); err != nil {
|
||||
return fmt.Errorf("snapshot database permissions: %w", err)
|
||||
}
|
||||
if err := syncFile(databasePath); err != nil {
|
||||
return fmt.Errorf("snapshot database sync: %w", err)
|
||||
}
|
||||
if afterDatabaseSnapshot != nil {
|
||||
if err := afterDatabaseSnapshot(); err != nil {
|
||||
return fmt.Errorf("snapshot test hook: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
snapshotDB, err := openSnapshotDatabase(databasePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
watermarks, err := collectSnapshotWatermarks(snapshotDB, Cfg.MboxRoot)
|
||||
if err != nil {
|
||||
_ = snapshotDB.Close()
|
||||
return err
|
||||
}
|
||||
stats, err := snapshotDatabaseStats(snapshotDB)
|
||||
if err == nil {
|
||||
stats.IndexSHA256, err = indexDigest(snapshotDB)
|
||||
}
|
||||
if closeErr := snapshotDB.Close(); err == nil && closeErr != nil {
|
||||
err = closeErr
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if stats.MissingArchiveRecords != 0 {
|
||||
return fmt.Errorf("snapshot refused: %d mbox_done identities have no archive index", stats.MissingArchiveRecords)
|
||||
}
|
||||
if stats.InvalidStageOrder != 0 {
|
||||
return fmt.Errorf("snapshot refused: %d precise identities have target_done=1 before mbox_done", stats.InvalidStageOrder)
|
||||
}
|
||||
var watermarkRecords int64
|
||||
for _, watermark := range watermarks {
|
||||
watermarkRecords += watermark.IndexRecords
|
||||
}
|
||||
if watermarkRecords != stats.IndexRecords {
|
||||
return fmt.Errorf("snapshot refused: indexed records=%d but mapped mbox records=%d", stats.IndexRecords, watermarkRecords)
|
||||
}
|
||||
|
||||
manifest := SnapshotManifest{
|
||||
FormatVersion: snapshotFormatVersion,
|
||||
CreatedAt: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
DatabasePath: databaseRel,
|
||||
IndexSHA256: stats.IndexSHA256,
|
||||
IndexRecords: stats.IndexRecords,
|
||||
PreciseCopied: stats.PreciseCopied,
|
||||
PendingMbox: stats.PendingMbox,
|
||||
PendingTarget: stats.PendingTarget,
|
||||
InvalidStageOrder: stats.InvalidStageOrder,
|
||||
MissingArchiveRecords: stats.MissingArchiveRecords,
|
||||
Files: make([]SnapshotFile, 0, len(watermarks)),
|
||||
}
|
||||
manifest.DatabaseSHA256, err = hashFile(databasePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("snapshot database hash: %w", err)
|
||||
}
|
||||
for _, watermark := range watermarks {
|
||||
destinationRel := filepath.ToSlash(filepath.Join("backup", watermark.RelativePath))
|
||||
destination := filepath.Join(partial, filepath.FromSlash(destinationRel))
|
||||
hash, err := copyFilePrefix(watermark.SourcePath, destination, watermark.Bytes)
|
||||
if err != nil {
|
||||
return fmt.Errorf("snapshot mbox %s: %w", destinationRel, err)
|
||||
}
|
||||
manifest.Files = append(manifest.Files, SnapshotFile{
|
||||
Path: destinationRel,
|
||||
Bytes: watermark.Bytes,
|
||||
SHA256: hash,
|
||||
AccountID: watermark.AccountID,
|
||||
Account: watermark.Account,
|
||||
Folder: watermark.Folder,
|
||||
IndexRecords: watermark.IndexRecords,
|
||||
})
|
||||
}
|
||||
sortSnapshotFiles(manifest.Files)
|
||||
manifestPath := filepath.Join(partial, "snapshot-manifest.json")
|
||||
manifestJSON, err := json.MarshalIndent(manifest, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("snapshot manifest encode: %w", err)
|
||||
}
|
||||
manifestJSON = append(manifestJSON, '\n')
|
||||
if err := os.WriteFile(manifestPath, manifestJSON, 0o600); err != nil {
|
||||
return fmt.Errorf("snapshot manifest write: %w", err)
|
||||
}
|
||||
if err := syncFile(manifestPath); err != nil {
|
||||
return fmt.Errorf("snapshot manifest sync: %w", err)
|
||||
}
|
||||
if err := os.Rename(partial, targetAbs); err != nil {
|
||||
return fmt.Errorf("snapshot finalize: %w", err)
|
||||
}
|
||||
complete = true
|
||||
log.Printf("snapshot complete target=%s files=%d bytes=%d index_records=%d precise_copied=%d pending_mbox=%d pending_target=%d",
|
||||
targetAbs, len(manifest.Files), snapshotBytes(manifest.Files), manifest.IndexRecords, manifest.PreciseCopied, manifest.PendingMbox, manifest.PendingTarget)
|
||||
return nil
|
||||
}
|
||||
|
||||
type snapshotStats struct {
|
||||
IndexRecords int64
|
||||
PreciseCopied int64
|
||||
PendingMbox int64
|
||||
PendingTarget int64
|
||||
MissingArchiveRecords int64
|
||||
InvalidStageOrder int64
|
||||
IndexSHA256 string
|
||||
}
|
||||
|
||||
func snapshotDatabaseStats(db *sql.DB) (snapshotStats, error) {
|
||||
var stats snapshotStats
|
||||
queries := []struct {
|
||||
query string
|
||||
dest *int64
|
||||
}{
|
||||
{`SELECT count(*) FROM mbox_index`, &stats.IndexRecords},
|
||||
{`SELECT count(*) FROM copied WHERE body_sha256<>''`, &stats.PreciseCopied},
|
||||
{`SELECT count(*) FROM copied WHERE body_sha256<>'' AND mbox_done=0`, &stats.PendingMbox},
|
||||
{`SELECT count(*) FROM copied WHERE body_sha256<>'' AND target_done=0`, &stats.PendingTarget},
|
||||
{`SELECT count(*) FROM copied WHERE body_sha256<>'' AND target_done=1 AND mbox_done=0`, &stats.InvalidStageOrder},
|
||||
{`SELECT count(*) FROM copied c WHERE c.body_sha256<>'' AND c.mbox_done=1 AND NOT EXISTS (
|
||||
SELECT 1 FROM mbox_index i
|
||||
WHERE i.account_id=c.account_id AND i.folder=c.folder
|
||||
AND i.message_id=c.message_id AND i.body_sha256=c.body_sha256
|
||||
)`, &stats.MissingArchiveRecords},
|
||||
}
|
||||
for _, item := range queries {
|
||||
if err := db.QueryRow(item.query).Scan(item.dest); err != nil {
|
||||
return snapshotStats{}, fmt.Errorf("snapshot database statistics: %w", err)
|
||||
}
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func indexDigest(db *sql.DB) (string, error) {
|
||||
rows, err := db.Query(`SELECT account_id, folder, seq, message_id, body_sha256
|
||||
FROM mbox_index ORDER BY account_id, folder, seq`)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("snapshot index digest query: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
hash := sha256.New()
|
||||
for rows.Next() {
|
||||
var accountID, seq int64
|
||||
var folder, messageID, bodyHash string
|
||||
if err := rows.Scan(&accountID, &folder, &seq, &messageID, &bodyHash); err != nil {
|
||||
return "", fmt.Errorf("snapshot index digest row: %w", err)
|
||||
}
|
||||
for _, value := range []int64{accountID, seq} {
|
||||
if err := binary.Write(hash, binary.LittleEndian, value); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
for _, value := range []string{folder, messageID, bodyHash} {
|
||||
if err := binary.Write(hash, binary.LittleEndian, uint64(len(value))); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := io.WriteString(hash, value); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return "", fmt.Errorf("snapshot index digest: %w", err)
|
||||
}
|
||||
return hex.EncodeToString(hash.Sum(nil)), nil
|
||||
}
|
||||
|
||||
func collectSnapshotWatermarks(db *sql.DB, mboxRoot string) ([]snapshotWatermark, error) {
|
||||
rootAbs, err := filepath.Abs(mboxRoot)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("snapshot mbox root: %w", err)
|
||||
}
|
||||
rows, err := db.Query(`SELECT i.account_id, a.name, a.mbox_dir, i.folder,
|
||||
MAX(i.file_offset+i.frame_len), count(*)
|
||||
FROM mbox_index i JOIN accounts a ON a.id=i.account_id
|
||||
GROUP BY i.account_id, a.name, a.mbox_dir, i.folder
|
||||
ORDER BY lower(a.name), a.name, i.folder`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("snapshot watermarks: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []snapshotWatermark
|
||||
seenPaths := map[string]bool{}
|
||||
archivePaths := map[string]map[string]string{}
|
||||
for rows.Next() {
|
||||
var watermark snapshotWatermark
|
||||
if err := rows.Scan(&watermark.AccountID, &watermark.Account, &watermark.MboxDir, &watermark.Folder, &watermark.Bytes, &watermark.IndexRecords); err != nil {
|
||||
return nil, fmt.Errorf("snapshot watermark row: %w", err)
|
||||
}
|
||||
watermark.MboxDir = strings.TrimSpace(watermark.MboxDir)
|
||||
if watermark.MboxDir == "" {
|
||||
return nil, fmt.Errorf("snapshot account %s has no mbox_dir", watermark.Account)
|
||||
}
|
||||
paths, ok := archivePaths[watermark.MboxDir]
|
||||
if !ok {
|
||||
paths = map[string]string{}
|
||||
for _, path := range archiveMboxPaths(filepath.Join(rootAbs, watermark.MboxDir)) {
|
||||
paths[folderNameFromMboxPath(path)] = path
|
||||
}
|
||||
archivePaths[watermark.MboxDir] = paths
|
||||
}
|
||||
watermark.SourcePath = paths[watermark.Folder]
|
||||
if watermark.SourcePath == "" {
|
||||
return nil, fmt.Errorf("snapshot indexed mbox missing for account=%s folder=%q", watermark.Account, watermark.Folder)
|
||||
}
|
||||
pathAbs, err := filepath.Abs(watermark.SourcePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("snapshot mbox path: %w", err)
|
||||
}
|
||||
relative, err := filepath.Rel(rootAbs, pathAbs)
|
||||
if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(os.PathSeparator)) || filepath.IsAbs(relative) {
|
||||
return nil, fmt.Errorf("snapshot mbox escapes configured root: %s", pathAbs)
|
||||
}
|
||||
watermark.SourcePath = pathAbs
|
||||
watermark.RelativePath = relative
|
||||
pathKey := snapshotPathKey(relative)
|
||||
if seenPaths[pathKey] {
|
||||
return nil, fmt.Errorf("snapshot multiple index groups resolve to the same mbox: %s", relative)
|
||||
}
|
||||
seenPaths[pathKey] = true
|
||||
info, err := os.Stat(pathAbs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("snapshot mbox stat %s: %w", pathAbs, err)
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return nil, fmt.Errorf("snapshot mbox is not a regular file: %s", pathAbs)
|
||||
}
|
||||
if watermark.Bytes <= 0 || info.Size() < watermark.Bytes {
|
||||
return nil, fmt.Errorf("snapshot invalid watermark account=%s folder=%q boundary=%d size=%d",
|
||||
watermark.Account, watermark.Folder, watermark.Bytes, info.Size())
|
||||
}
|
||||
out = append(out, watermark)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("snapshot watermarks: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func copyFilePrefix(source, destination string, length int64) (string, error) {
|
||||
if length < 0 {
|
||||
return "", fmt.Errorf("negative prefix length %d", length)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(destination), 0o700); err != nil {
|
||||
return "", err
|
||||
}
|
||||
src, err := os.Open(source)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer src.Close()
|
||||
dst, err := os.OpenFile(destination, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
hash := sha256.New()
|
||||
written, copyErr := io.CopyN(io.MultiWriter(dst, hash), src, length)
|
||||
if copyErr == nil && written != length {
|
||||
copyErr = io.ErrShortWrite
|
||||
}
|
||||
if copyErr == nil {
|
||||
copyErr = dst.Sync()
|
||||
}
|
||||
closeErr := dst.Close()
|
||||
if copyErr != nil {
|
||||
return "", copyErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return "", closeErr
|
||||
}
|
||||
return hex.EncodeToString(hash.Sum(nil)), nil
|
||||
}
|
||||
|
||||
func openSnapshotDatabase(path string) (*sql.DB, error) {
|
||||
db, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open snapshot database: %w", err)
|
||||
}
|
||||
db.SetMaxOpenConns(1)
|
||||
if _, err := db.Exec(`PRAGMA query_only=ON; PRAGMA foreign_keys=ON;`); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, fmt.Errorf("open snapshot database read-only: %w", err)
|
||||
}
|
||||
return db, nil
|
||||
}
|
||||
|
||||
// VerifySnapshot validates hashes and DB/file boundaries, then copies only the
|
||||
// small DB into a temporary restore workspace and performs a full reindex there.
|
||||
// The snapshot mbox files and any live database remain untouched.
|
||||
func VerifySnapshot(root string) error {
|
||||
root = strings.TrimSpace(root)
|
||||
if root == "" {
|
||||
return fmt.Errorf("verify snapshot: directory is empty")
|
||||
}
|
||||
rootAbs, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return fmt.Errorf("verify snapshot path: %w", err)
|
||||
}
|
||||
manifestBytes, err := os.ReadFile(filepath.Join(rootAbs, "snapshot-manifest.json"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("verify snapshot manifest: %w", err)
|
||||
}
|
||||
var manifest SnapshotManifest
|
||||
if err := json.Unmarshal(manifestBytes, &manifest); err != nil {
|
||||
return fmt.Errorf("verify snapshot manifest decode: %w", err)
|
||||
}
|
||||
if manifest.FormatVersion != snapshotFormatVersion {
|
||||
return fmt.Errorf("verify snapshot format=%d, supported=%d", manifest.FormatVersion, snapshotFormatVersion)
|
||||
}
|
||||
databasePath, err := pathInsideSnapshot(rootAbs, manifest.DatabasePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if runtime.GOOS != "windows" {
|
||||
databaseInfo, err := os.Stat(databasePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("verify snapshot database permissions: %w", err)
|
||||
}
|
||||
if permissions := databaseInfo.Mode().Perm(); permissions&0o077 != 0 {
|
||||
return fmt.Errorf("verify snapshot database permissions=%#o, want no group/other access", permissions)
|
||||
}
|
||||
}
|
||||
databaseHash, err := hashFile(databasePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("verify snapshot database hash: %w", err)
|
||||
}
|
||||
if databaseHash != manifest.DatabaseSHA256 {
|
||||
return fmt.Errorf("verify snapshot database hash mismatch: got %s want %s", databaseHash, manifest.DatabaseSHA256)
|
||||
}
|
||||
for _, file := range manifest.Files {
|
||||
path, err := pathInsideSnapshot(rootAbs, file.Path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("verify snapshot file %s: %w", file.Path, err)
|
||||
}
|
||||
if info.Size() != file.Bytes {
|
||||
return fmt.Errorf("verify snapshot file size %s: got %d want %d", file.Path, info.Size(), file.Bytes)
|
||||
}
|
||||
hash, err := hashFile(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("verify snapshot file hash %s: %w", file.Path, err)
|
||||
}
|
||||
if hash != file.SHA256 {
|
||||
return fmt.Errorf("verify snapshot file hash mismatch %s: got %s want %s", file.Path, hash, file.SHA256)
|
||||
}
|
||||
}
|
||||
|
||||
snapshotDB, err := openSnapshotDatabase(databasePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var integrity string
|
||||
if err := snapshotDB.QueryRow(`PRAGMA integrity_check`).Scan(&integrity); err != nil {
|
||||
_ = snapshotDB.Close()
|
||||
return fmt.Errorf("verify snapshot integrity: %w", err)
|
||||
}
|
||||
if integrity != "ok" {
|
||||
_ = snapshotDB.Close()
|
||||
return fmt.Errorf("verify snapshot integrity: %s", integrity)
|
||||
}
|
||||
stats, err := snapshotDatabaseStats(snapshotDB)
|
||||
if err == nil {
|
||||
stats.IndexSHA256, err = indexDigest(snapshotDB)
|
||||
}
|
||||
if err != nil {
|
||||
_ = snapshotDB.Close()
|
||||
return err
|
||||
}
|
||||
watermarks, err := collectSnapshotWatermarks(snapshotDB, filepath.Join(rootAbs, "backup"))
|
||||
if closeErr := snapshotDB.Close(); err == nil && closeErr != nil {
|
||||
err = closeErr
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if stats.IndexRecords != manifest.IndexRecords || stats.PreciseCopied != manifest.PreciseCopied ||
|
||||
stats.PendingMbox != manifest.PendingMbox || stats.PendingTarget != manifest.PendingTarget ||
|
||||
stats.InvalidStageOrder != manifest.InvalidStageOrder || stats.MissingArchiveRecords != manifest.MissingArchiveRecords ||
|
||||
stats.IndexSHA256 != manifest.IndexSHA256 {
|
||||
return fmt.Errorf("verify snapshot manifest/database statistics differ")
|
||||
}
|
||||
if stats.MissingArchiveRecords != 0 {
|
||||
return fmt.Errorf("verify snapshot has %d mbox_done identities without archive index", stats.MissingArchiveRecords)
|
||||
}
|
||||
if stats.InvalidStageOrder != 0 {
|
||||
return fmt.Errorf("verify snapshot has %d precise identities with invalid stage order", stats.InvalidStageOrder)
|
||||
}
|
||||
if err := compareManifestWatermarks(manifest.Files, watermarks); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := verifySnapshotReindex(rootAbs, databasePath, manifest.IndexRecords, manifest.IndexSHA256); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("snapshot verified root=%s files=%d bytes=%d index_records=%d precise_copied=%d pending_mbox=%d pending_target=%d",
|
||||
rootAbs, len(manifest.Files), snapshotBytes(manifest.Files), manifest.IndexRecords, manifest.PreciseCopied, manifest.PendingMbox, manifest.PendingTarget)
|
||||
return nil
|
||||
}
|
||||
|
||||
func compareManifestWatermarks(files []SnapshotFile, watermarks []snapshotWatermark) error {
|
||||
manifestByPath := make(map[string]SnapshotFile, len(files))
|
||||
for _, file := range files {
|
||||
key := snapshotPathKey(filepath.FromSlash(file.Path))
|
||||
if _, exists := manifestByPath[key]; exists {
|
||||
return fmt.Errorf("verify snapshot duplicate manifest path: %s", file.Path)
|
||||
}
|
||||
manifestByPath[key] = file
|
||||
}
|
||||
if len(manifestByPath) != len(watermarks) {
|
||||
return fmt.Errorf("verify snapshot file/watermark count differs: manifest=%d database=%d", len(manifestByPath), len(watermarks))
|
||||
}
|
||||
for _, watermark := range watermarks {
|
||||
path := filepath.Join("backup", watermark.RelativePath)
|
||||
key := snapshotPathKey(path)
|
||||
file, ok := manifestByPath[key]
|
||||
if !ok {
|
||||
return fmt.Errorf("verify snapshot manifest missing DB watermark: %s", filepath.ToSlash(path))
|
||||
}
|
||||
if file.Bytes != watermark.Bytes || file.IndexRecords != watermark.IndexRecords || file.AccountID != watermark.AccountID || file.Folder != watermark.Folder {
|
||||
return fmt.Errorf("verify snapshot watermark mismatch: %s", file.Path)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func verifySnapshotReindex(root, databasePath string, expectedIndexRecords int64, expectedIndexSHA256 string) error {
|
||||
temporaryRoot, err := os.MkdirTemp("", "mail-graveyard-snapshot-verify-")
|
||||
if err != nil {
|
||||
return fmt.Errorf("verify snapshot temporary restore: %w", err)
|
||||
}
|
||||
defer os.RemoveAll(temporaryRoot)
|
||||
temporaryDB := filepath.Join(temporaryRoot, "mail-graveyard.db")
|
||||
databaseInfo, err := os.Stat(databasePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("verify snapshot temporary database stat: %w", err)
|
||||
}
|
||||
if _, err := copyFilePrefix(databasePath, temporaryDB, databaseInfo.Size()); err != nil {
|
||||
return fmt.Errorf("verify snapshot temporary database copy: %w", err)
|
||||
}
|
||||
oldCfg, oldDB := Cfg, DB
|
||||
Cfg.DBPath = temporaryDB
|
||||
Cfg.MboxRoot = filepath.Join(root, "backup")
|
||||
DB = nil
|
||||
defer func() {
|
||||
if DB != nil {
|
||||
_ = DB.Close()
|
||||
}
|
||||
Cfg, DB = oldCfg, oldDB
|
||||
}()
|
||||
if err := ConnectDB(false); err != nil {
|
||||
return fmt.Errorf("verify snapshot restore database: %w", err)
|
||||
}
|
||||
if _, err := DB.Exec(`CREATE TABLE snapshot_verify_expected_index AS
|
||||
SELECT account_id, folder, seq, message_id, body_sha256 FROM mbox_index`); err != nil {
|
||||
return fmt.Errorf("verify snapshot preserve expected index: %w", err)
|
||||
}
|
||||
if err := ReindexArchives("all"); err != nil {
|
||||
return fmt.Errorf("verify snapshot restore reindex: %w", err)
|
||||
}
|
||||
var reindexed int64
|
||||
if err := DB.QueryRow(`SELECT count(*) FROM mbox_index`).Scan(&reindexed); err != nil {
|
||||
return fmt.Errorf("verify snapshot restore index count: %w", err)
|
||||
}
|
||||
if reindexed != expectedIndexRecords {
|
||||
return fmt.Errorf("verify snapshot restore index count=%d, want %d", reindexed, expectedIndexRecords)
|
||||
}
|
||||
reindexedDigest, err := indexDigest(DB)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if reindexedDigest != expectedIndexSHA256 {
|
||||
difference, differenceErr := snapshotIndexDifference(DB)
|
||||
if differenceErr != nil {
|
||||
return fmt.Errorf("verify snapshot restore index digest=%s, want %s (difference query: %v)", reindexedDigest, expectedIndexSHA256, differenceErr)
|
||||
}
|
||||
return fmt.Errorf("verify snapshot restore index digest=%s, want %s (%s)", reindexedDigest, expectedIndexSHA256, difference)
|
||||
}
|
||||
stats, err := snapshotDatabaseStats(DB)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if stats.MissingArchiveRecords != 0 {
|
||||
return fmt.Errorf("verify snapshot restore has %d copied identities without archive index", stats.MissingArchiveRecords)
|
||||
}
|
||||
if stats.InvalidStageOrder != 0 {
|
||||
return fmt.Errorf("verify snapshot restore has %d precise identities with invalid stage order", stats.InvalidStageOrder)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func snapshotIndexDifference(db *sql.DB) (string, error) {
|
||||
type differenceRow struct {
|
||||
accountID int64
|
||||
folder string
|
||||
seq int64
|
||||
messageID string
|
||||
bodyHash string
|
||||
}
|
||||
queries := []struct {
|
||||
label string
|
||||
left string
|
||||
right string
|
||||
}{
|
||||
{"missing after reindex", "snapshot_verify_expected_index", "mbox_index"},
|
||||
{"unexpected after reindex", "mbox_index", "snapshot_verify_expected_index"},
|
||||
}
|
||||
parts := make([]string, 0, len(queries))
|
||||
for _, query := range queries {
|
||||
statement := fmt.Sprintf(`SELECT account_id, folder, seq, message_id, body_sha256 FROM %s
|
||||
EXCEPT SELECT account_id, folder, seq, message_id, body_sha256 FROM %s`, query.left, query.right)
|
||||
var count int64
|
||||
if err := db.QueryRow(`SELECT count(*) FROM (` + statement + `)`).Scan(&count); err != nil {
|
||||
return "", err
|
||||
}
|
||||
rows, err := db.Query(statement + ` LIMIT 3`)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var samples []string
|
||||
for rows.Next() {
|
||||
var row differenceRow
|
||||
if err := rows.Scan(&row.accountID, &row.folder, &row.seq, &row.messageID, &row.bodyHash); err != nil {
|
||||
_ = rows.Close()
|
||||
return "", err
|
||||
}
|
||||
hash := row.bodyHash
|
||||
if len(hash) > 16 {
|
||||
hash = hash[:16]
|
||||
}
|
||||
samples = append(samples, fmt.Sprintf("account=%d folder=%q seq=%d id=%q hash=%s", row.accountID, row.folder, row.seq, row.messageID, hash))
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
_ = rows.Close()
|
||||
return "", err
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
parts = append(parts, fmt.Sprintf("%s=%d samples=[%s]", query.label, count, strings.Join(samples, "; ")))
|
||||
}
|
||||
return strings.Join(parts, ", "), nil
|
||||
}
|
||||
|
||||
func pathInsideSnapshot(root, relative string) (string, error) {
|
||||
if relative == "" || filepath.IsAbs(relative) {
|
||||
return "", fmt.Errorf("verify snapshot invalid relative path: %q", relative)
|
||||
}
|
||||
path := filepath.Join(root, filepath.FromSlash(relative))
|
||||
pathAbs, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
rel, err := filepath.Rel(root, pathAbs)
|
||||
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
|
||||
return "", fmt.Errorf("verify snapshot path escapes root: %q", relative)
|
||||
}
|
||||
return pathAbs, nil
|
||||
}
|
||||
|
||||
func hashFile(path string) (string, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
hash := sha256.New()
|
||||
if _, err := io.Copy(hash, f); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(hash.Sum(nil)), nil
|
||||
}
|
||||
|
||||
func syncFile(path string) error {
|
||||
f, err := os.OpenFile(path, os.O_RDWR, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
return f.Sync()
|
||||
}
|
||||
|
||||
func snapshotBytes(files []SnapshotFile) int64 {
|
||||
var total int64
|
||||
for _, file := range files {
|
||||
total += file.Bytes
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func snapshotPathKey(path string) string {
|
||||
path = filepath.Clean(path)
|
||||
if runtime.GOOS == "windows" {
|
||||
return strings.ToLower(path)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func sortSnapshotFiles(files []SnapshotFile) {
|
||||
sort.Slice(files, func(i, j int) bool { return files[i].Path < files[j].Path })
|
||||
}
|
||||
209
backend/14-snapshot_test.go
Normal file
209
backend/14-snapshot_test.go
Normal file
|
|
@ -0,0 +1,209 @@
|
|||
package backend
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSnapshotUsesDatabaseWatermarkWhileArchiveKeepsGrowing(t *testing.T) {
|
||||
for _, compression := range []string{"none", "zstd"} {
|
||||
t.Run(compression, func(t *testing.T) {
|
||||
account, writer, restore := setupSnapshotTest(t, compression)
|
||||
defer restore()
|
||||
|
||||
first := RawMessage{
|
||||
MessageID: "first@example.com",
|
||||
Body: []byte("Message-ID: <first@example.com>\r\nSubject: First\r\n\r\nFirst body\r\nFrom quoted\r\n"),
|
||||
InternalDate: time.Unix(1_700_000_000, 0),
|
||||
}
|
||||
firstInfo := appendSnapshotTestMessage(t, account, writer, "INBOX", first)
|
||||
snapshotRoot := filepath.Join(t.TempDir(), "snapshot")
|
||||
var secondInfo MboxAppendInfo
|
||||
err := createSnapshot(snapshotRoot, func() error {
|
||||
second := RawMessage{
|
||||
MessageID: "second@example.com",
|
||||
Body: []byte("Message-ID: <second@example.com>\r\nSubject: Second\r\n\r\nSecond body"),
|
||||
InternalDate: time.Unix(1_700_000_100, 0),
|
||||
}
|
||||
secondInfo = appendSnapshotTestMessage(t, account, writer, "INBOX", second)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
manifest := readSnapshotTestManifest(t, snapshotRoot)
|
||||
if runtime.GOOS != "windows" {
|
||||
databaseInfo, err := os.Stat(filepath.Join(snapshotRoot, filepath.FromSlash(manifest.DatabasePath)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if permissions := databaseInfo.Mode().Perm(); permissions != 0o600 {
|
||||
t.Fatalf("snapshot database permissions=%#o, want 0600", permissions)
|
||||
}
|
||||
}
|
||||
if manifest.IndexRecords != 1 || manifest.PreciseCopied != 1 || len(manifest.Files) != 1 {
|
||||
t.Fatalf("snapshot crossed DB boundary: %#v", manifest)
|
||||
}
|
||||
if manifest.Files[0].Bytes != firstInfo.FileOffset+firstInfo.FrameLen {
|
||||
t.Fatalf("snapshot bytes=%d, want first committed boundary=%d", manifest.Files[0].Bytes, firstInfo.FileOffset+firstInfo.FrameLen)
|
||||
}
|
||||
sourceInfo, err := os.Stat(secondInfo.Path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sourceInfo.Size() <= manifest.Files[0].Bytes {
|
||||
t.Fatalf("source did not grow beyond snapshot boundary: source=%d snapshot=%d", sourceInfo.Size(), manifest.Files[0].Bytes)
|
||||
}
|
||||
snapshotMbox := filepath.Join(snapshotRoot, filepath.FromSlash(manifest.Files[0].Path))
|
||||
messages, err := readMboxMessages(snapshotMbox)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(messages) != 1 || !strings.Contains(string(messages[0]), "First body") {
|
||||
t.Fatalf("snapshot messages=%d, expected only first message", len(messages))
|
||||
}
|
||||
if err := VerifySnapshot(snapshotRoot); err != nil {
|
||||
t.Fatalf("verify valid snapshot: %v", err)
|
||||
}
|
||||
var liveIndex int
|
||||
if err := DB.QueryRow(`SELECT count(*) FROM mbox_index WHERE account_id=?`, account.ID).Scan(&liveIndex); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if liveIndex != 2 {
|
||||
t.Fatalf("snapshot verification altered live database: index=%d", liveIndex)
|
||||
}
|
||||
|
||||
f, err := os.OpenFile(snapshotMbox, os.O_APPEND|os.O_WRONLY, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := f.Write([]byte("tamper")); err != nil {
|
||||
_ = f.Close()
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := VerifySnapshot(snapshotRoot); err == nil {
|
||||
t.Fatal("verification accepted a modified mbox")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshotRefusesKnownMissingArchiveIdentity(t *testing.T) {
|
||||
account, _, restore := setupSnapshotTest(t, "none")
|
||||
defer restore()
|
||||
identity := MessageIdentity{MessageID: "missing@example.com", BodySHA256: bodySHA256([]byte("missing"))}
|
||||
if err := MarkIdentityCopied(account.ID, "INBOX", identity); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
target := filepath.Join(t.TempDir(), "refused")
|
||||
err := CreateSnapshot(target)
|
||||
if err == nil || !strings.Contains(err.Error(), "no archive index") {
|
||||
t.Fatalf("snapshot error=%v, want missing archive refusal", err)
|
||||
}
|
||||
if _, statErr := os.Stat(target); !os.IsNotExist(statErr) {
|
||||
t.Fatalf("refused snapshot unexpectedly finalized: %v", statErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshotDoesNotOverwriteExistingTarget(t *testing.T) {
|
||||
_, _, restore := setupSnapshotTest(t, "none")
|
||||
defer restore()
|
||||
target := filepath.Join(t.TempDir(), "existing")
|
||||
if err := os.Mkdir(target, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sentinel := filepath.Join(target, "keep")
|
||||
if err := os.WriteFile(sentinel, []byte("keep"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := CreateSnapshot(target); err == nil {
|
||||
t.Fatal("snapshot overwrote an existing target")
|
||||
}
|
||||
if got, err := os.ReadFile(sentinel); err != nil || string(got) != "keep" {
|
||||
t.Fatalf("existing target changed: %q %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func setupSnapshotTest(t *testing.T, compression string) (Account, *MboxWriter, func()) {
|
||||
t.Helper()
|
||||
oldCfg, oldDB := Cfg, DB
|
||||
root := t.TempDir()
|
||||
Cfg = Config{
|
||||
DBPath: filepath.Join(root, "mail-graveyard.db"),
|
||||
MboxRoot: filepath.Join(root, "backup"),
|
||||
MboxCompression: compression,
|
||||
}
|
||||
DB = nil
|
||||
if err := ConnectDB(true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
restore := func() {
|
||||
if DB != nil {
|
||||
_ = DB.Close()
|
||||
}
|
||||
Cfg, DB = oldCfg, oldDB
|
||||
}
|
||||
if err := SaveAccount(Account{
|
||||
Name: "snapshot-account", SrcHost: "source.example", SrcPort: 993, SrcSecurity: "tls", SrcUser: "source@example.com", SrcPass: "x",
|
||||
DstHost: "target.example", DstPort: 993, DstSecurity: "tls", DstUser: "target@example.com", DstPass: "x",
|
||||
MboxDir: "snapshot@example.com", Active: true,
|
||||
}); err != nil {
|
||||
restore()
|
||||
t.Fatal(err)
|
||||
}
|
||||
account, err := GetAccount("snapshot-account")
|
||||
if err != nil {
|
||||
restore()
|
||||
t.Fatal(err)
|
||||
}
|
||||
writer, err := NewMboxWriter(filepath.Join(Cfg.MboxRoot, account.MboxDir))
|
||||
if err != nil {
|
||||
restore()
|
||||
t.Fatal(err)
|
||||
}
|
||||
return account, writer, restore
|
||||
}
|
||||
|
||||
func appendSnapshotTestMessage(t *testing.T, account Account, writer *MboxWriter, folder string, message RawMessage) MboxAppendInfo {
|
||||
t.Helper()
|
||||
identity := identityForRawMessage(message)
|
||||
info, err := writer.Append(folder, message)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SaveMboxIndex(MboxIndexEntry{
|
||||
AccountID: account.ID, Folder: safeMboxName(folder), MessageID: identity.MessageID, BodySHA256: identity.BodySHA256,
|
||||
FileOffset: info.FileOffset, FrameLen: info.FrameLen, InnerOffset: info.InnerOffset, InnerLen: info.InnerLen,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := UpdateMboxIndexState(account.ID, safeMboxName(folder), info.FileOffset+info.FrameLen); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := MarkIdentityCopied(account.ID, safeMboxName(folder), identity); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return info
|
||||
}
|
||||
|
||||
func readSnapshotTestManifest(t *testing.T, root string) SnapshotManifest {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(filepath.Join(root, "snapshot-manifest.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var manifest SnapshotManifest
|
||||
if err := json.Unmarshal(b, &manifest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return manifest
|
||||
}
|
||||
16
main.go
16
main.go
|
|
@ -25,16 +25,32 @@ func main() {
|
|||
checkAccount := flag.String("check-account", "", "Konto-Logins pruefen und INBOX zaehlen, ohne zu kopieren")
|
||||
reindex := flag.String("reindex", "", "Archiv-mbox-Index fuer Konto/Archiv oder 'all' neu aufbauen")
|
||||
dedupTarget := flag.String("dedup-target", "", "Ziel-Postfach nach Message-ID-Dubletten scannen (Dry-Run ohne --apply)")
|
||||
snapshot := flag.String("snapshot", "", "konsistenten DB- und mbox-Praefix-Snapshot in ein neues Zielverzeichnis schreiben")
|
||||
verifySnapshot := flag.String("verify-snapshot", "", "Snapshot isoliert pruefen und dessen DB-Kopie vollstaendig reindexieren")
|
||||
apply := flag.Bool("apply", false, "Scharfe Ausfuehrung fuer riskante Wartungsbefehle wie --dedup-target")
|
||||
initDB := flag.Bool("init-db", false, "SQLite-DB neu anlegen, falls db_path noch nicht existiert")
|
||||
flag.Parse()
|
||||
if *snapshot != "" && (*verifySnapshot != "" || *runOnce != "" || *seedAccount != "" || *checkAccount != "" || *reindex != "" || *dedupTarget != "" || *watch) {
|
||||
log.Fatal("-snapshot kann nicht mit einem anderen Betriebsmodus kombiniert werden")
|
||||
}
|
||||
if *verifySnapshot != "" && (*runOnce != "" || *seedAccount != "" || *checkAccount != "" || *reindex != "" || *dedupTarget != "" || *watch) {
|
||||
log.Fatal("-verify-snapshot kann nicht mit einem anderen Betriebsmodus kombiniert werden")
|
||||
}
|
||||
|
||||
if err := backend.LoadConfig("config.json"); err != nil {
|
||||
log.Fatalf("config: %v", err)
|
||||
}
|
||||
if *verifySnapshot != "" {
|
||||
must(backend.VerifySnapshot(*verifySnapshot))
|
||||
return
|
||||
}
|
||||
if err := backend.ConnectDB(*initDB); err != nil {
|
||||
log.Fatalf("database: %v", err)
|
||||
}
|
||||
if *snapshot != "" {
|
||||
must(backend.CreateSnapshot(*snapshot))
|
||||
return
|
||||
}
|
||||
if err := backend.InitAuth(); err != nil {
|
||||
log.Fatalf("auth: %v", err)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue