Add verified snapshots and harden archive identity
This commit is contained in:
parent
c6de214ac8
commit
68b9cee880
9 changed files with 1372 additions and 52 deletions
34
INSTALL.md
34
INSTALL.md
|
|
@ -47,8 +47,10 @@ Ohne Web, z. B. als Cronjob oder Dauerlauf im Umzugsfenster:
|
||||||
./mail-graveyard --run dr-gold --watch # ein Konto, Delta-Schleife bis Stop
|
./mail-graveyard --run dr-gold --watch # ein Konto, Delta-Schleife bis Stop
|
||||||
```
|
```
|
||||||
|
|
||||||
Dank Message-ID-Cache kopiert jeder Lauf nur Neues — beliebig oft wiederholbar
|
Die kanonische Inhaltsidentitaet verhindert dabei doppelte **Schreibvorgaenge**
|
||||||
ohne Dubletten. Für den Dauerlauf reicht `nohup`/systemd/Aufgabenplanung.
|
in Archiv und Ziel. Der aktuelle Volllauf laedt trotzdem jede Quellmail erneut,
|
||||||
|
bevor er deren Body-Hash pruefen kann. Grosse Konten deshalb bis zum
|
||||||
|
UID-Delta-Watcher nicht in kurzen Intervallen als Dauerlauf starten.
|
||||||
|
|
||||||
## 5. Wartung
|
## 5. Wartung
|
||||||
|
|
||||||
|
|
@ -56,18 +58,46 @@ ohne Dubletten. Für den Dauerlauf reicht `nohup`/systemd/Aufgabenplanung.
|
||||||
./mail-graveyard --reindex all
|
./mail-graveyard --reindex all
|
||||||
./mail-graveyard --dedup-target konto-name # Dry-Run
|
./mail-graveyard --dedup-target konto-name # Dry-Run
|
||||||
./mail-graveyard --dedup-target konto-name --apply # loescht Dubletten im Ziel
|
./mail-graveyard --dedup-target konto-name --apply # loescht Dubletten im Ziel
|
||||||
|
./mail-graveyard --snapshot /sicherungen/stand-2026-07-16
|
||||||
|
./mail-graveyard --verify-snapshot /sicherungen/stand-2026-07-16
|
||||||
```
|
```
|
||||||
|
|
||||||
`--reindex` baut den Archiv-Index fuer vorhandene mbox/mbox.zst-Dateien neu auf.
|
`--reindex` baut den Archiv-Index fuer vorhandene mbox/mbox.zst-Dateien neu auf.
|
||||||
`--dedup-target` ist bewusst ein Dry-Run, bis `--apply` gesetzt wird; Mails ohne
|
`--dedup-target` ist bewusst ein Dry-Run, bis `--apply` gesetzt wird; Mails ohne
|
||||||
Message-ID werden nie geloescht.
|
Message-ID werden nie geloescht.
|
||||||
|
|
||||||
|
`--snapshot` erzeugt zuerst per SQLite `VACUUM INTO` einen konsistenten
|
||||||
|
DB-Stand. Anschliessend kopiert es aus jeder append-only mbox exakt das durch
|
||||||
|
`MAX(file_offset+frame_len)` **dieses DB-Standes** belegte Praefix. Das Ziel
|
||||||
|
muss neu sein und enthaelt `data/mail-graveyard.db`, `backup/...` sowie ein
|
||||||
|
Manifest mit Grenzen und SHA-256-Pruefsummen. Ein Fehler hinterlaesst ein klar
|
||||||
|
benanntes `.partial-*`-Verzeichnis und wird nie als fertiger Stand umbenannt.
|
||||||
|
Die Datenbank wird wegen der enthaltenen Klartext-Zugangsdaten mit `0600`
|
||||||
|
geschrieben; die Verifikation weist unter Linux weiter gefasste Rechte zurueck.
|
||||||
|
|
||||||
|
`--verify-snapshot` veraendert weder Produktion noch Snapshot: Es prueft
|
||||||
|
Manifest, Dateigroessen, SHA-256, SQLite-Integritaet, Stufenreihenfolge und den
|
||||||
|
Identitaetsdigest. Danach kopiert es nur die Snapshot-DB in ein temporaeres
|
||||||
|
Restore-Verzeichnis und reindiziert dort alle mbox-Dateien. Der
|
||||||
|
Identitaetsdigest muss vor und nach dem Reindex gleich bleiben.
|
||||||
|
|
||||||
|
Der lokale Snapshot ist nur die konsistente **Vorstufe** eines Backups. Er
|
||||||
|
enthaelt Kontozugangsdaten und muss anschliessend verschluesselt auf einen
|
||||||
|
raeumlich getrennten Datentraeger uebertragen werden.
|
||||||
|
|
||||||
## 6. Sicherheit
|
## 6. Sicherheit
|
||||||
|
|
||||||
Tool hält fremde IMAP-Passwörter und kann Mails senden → Default bind
|
Tool hält fremde IMAP-Passwörter und kann Mails senden → Default bind
|
||||||
`127.0.0.1`. Falls remote nötig: hinter Caddy + `/vadmin`-Mail-2FA. `config.json`
|
`127.0.0.1`. Falls remote nötig: hinter Caddy + `/vadmin`-Mail-2FA. `config.json`
|
||||||
und `*.db` nie committen.
|
und `*.db` nie committen.
|
||||||
|
|
||||||
|
Die Live-SQLite-Datei sowie ihre WAL-/SHM-Sidecars werden bei jedem Start auf
|
||||||
|
`0600` gesetzt. Das gilt auch fuer weitere `*.db`, `*.db-wal` und `*.db-shm`
|
||||||
|
direkt im konfigurierten Datenverzeichnis sowie fuer die Datenbank eines neu
|
||||||
|
erzeugten Snapshots. Kann ein Dateisystem Unix-Rechte nicht abbilden (z. B.
|
||||||
|
bestimmte exFAT-/CIFS-Mounts), startet das Werkzeug bewusst nicht: Zugangsdaten
|
||||||
|
bleiben fail-closed statt unbemerkt lesbar.
|
||||||
|
|
||||||
## 7. Reihenfolge bei einem echten Umzug
|
## 7. Reihenfolge bei einem echten Umzug
|
||||||
|
|
||||||
1. Ziel-Postfächer beim neuen Hoster anlegen (`archiv-…`, `temp-…`).
|
1. Ziel-Postfächer beim neuen Hoster anlegen (`archiv-…`, `temp-…`).
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,8 @@ import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
_ "modernc.org/sqlite"
|
_ "modernc.org/sqlite"
|
||||||
|
|
@ -85,6 +87,10 @@ func ConnectDB(initDB bool) error {
|
||||||
_ = db.Close()
|
_ = db.Close()
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if err := secureSQLiteFiles(Cfg.DBPath); err != nil {
|
||||||
|
_ = db.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
schema := []string{
|
schema := []string{
|
||||||
`CREATE TABLE IF NOT EXISTS accounts(
|
`CREATE TABLE IF NOT EXISTS accounts(
|
||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
|
@ -361,7 +367,7 @@ func ensureDBFile(path string, initDB bool) error {
|
||||||
if info.IsDir() {
|
if info.IsDir() {
|
||||||
return fmt.Errorf("DB %q ist ein Verzeichnis", path)
|
return fmt.Errorf("DB %q ist ein Verzeichnis", path)
|
||||||
}
|
}
|
||||||
return nil
|
return chmodPrivateFile(path)
|
||||||
}
|
}
|
||||||
if !errors.Is(err, os.ErrNotExist) {
|
if !errors.Is(err, os.ErrNotExist) {
|
||||||
return err
|
return err
|
||||||
|
|
@ -373,7 +379,57 @@ func ensureDBFile(path string, initDB bool) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return f.Close()
|
if err := f.Close(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return chmodPrivateFile(path)
|
||||||
|
}
|
||||||
|
|
||||||
|
func secureSQLiteFiles(path string) error {
|
||||||
|
directory := filepath.Dir(path)
|
||||||
|
entries, err := os.ReadDir(directory)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("scan SQLite directory %q: %w", directory, err)
|
||||||
|
}
|
||||||
|
for _, entry := range entries {
|
||||||
|
if entry.Type()&os.ModeSymlink != 0 || !isSQLiteDataFile(entry.Name()) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
info, err := entry.Info()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("inspect SQLite file %q: %w", filepath.Join(directory, entry.Name()), err)
|
||||||
|
}
|
||||||
|
if !info.Mode().IsRegular() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
candidate := filepath.Join(directory, entry.Name())
|
||||||
|
if err := chmodPrivateFile(candidate); err != nil {
|
||||||
|
return fmt.Errorf("secure SQLite file %q: %w", candidate, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func chmodPrivateFile(path string) error {
|
||||||
|
if err := os.Chmod(path, 0o600); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if permissions := info.Mode().Perm(); permissions&0o077 != 0 {
|
||||||
|
return fmt.Errorf("permissions=%#o after chmod 0600", permissions)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func isSQLiteDataFile(name string) bool {
|
||||||
|
name = strings.ToLower(name)
|
||||||
|
return strings.HasSuffix(name, ".db") || strings.HasSuffix(name, ".db-wal") || strings.HasSuffix(name, ".db-shm")
|
||||||
}
|
}
|
||||||
|
|
||||||
func ensureAppUserColumns() error {
|
func ensureAppUserColumns() error {
|
||||||
|
|
@ -724,56 +780,72 @@ func GetCopyIdentityStateWithFolderAlias(accountID int64, folder, legacyFolder s
|
||||||
if legacyFolder != "" && legacyFolder != folder {
|
if legacyFolder != "" && legacyFolder != folder {
|
||||||
folders = append(folders, legacyFolder)
|
folders = append(folders, legacyFolder)
|
||||||
}
|
}
|
||||||
|
aliases := []string{identity.MessageID, identity.LegacyMessageID}
|
||||||
|
aliases = append(aliases, identity.LegacyMessageIDs...)
|
||||||
|
if identity.MessageID == "" && identity.BodySHA256 != "" {
|
||||||
|
aliases = append(aliases, "sha256:"+identity.BodySHA256)
|
||||||
|
}
|
||||||
|
uniqueAliases := make([]string, 0, len(aliases))
|
||||||
|
seenAliases := map[string]bool{}
|
||||||
|
for _, alias := range aliases {
|
||||||
|
alias = strings.TrimSpace(alias)
|
||||||
|
if alias == "" || seenAliases[alias] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seenAliases[alias] = true
|
||||||
|
uniqueAliases = append(uniqueAliases, alias)
|
||||||
|
}
|
||||||
// Precise rows always win. For Message-ID-less mail the hash is the whole
|
// Precise rows always win. For Message-ID-less mail the hash is the whole
|
||||||
// identity, irrespective of which legacy key happened to be stored beside it.
|
// identity, irrespective of which legacy key happened to be stored beside it.
|
||||||
|
// For mail with an ID, historical normalizations are precise aliases only
|
||||||
|
// when the canonical body hash is identical.
|
||||||
var mboxDone, targetDone int
|
var mboxDone, targetDone int
|
||||||
for _, candidateFolder := range folders {
|
for _, candidateFolder := range folders {
|
||||||
var mbox, target int
|
|
||||||
var err error
|
|
||||||
if identity.MessageID == "" {
|
if identity.MessageID == "" {
|
||||||
err = DB.QueryRow(`SELECT COALESCE(MAX(mbox_done),0), COALESCE(MAX(target_done),0) FROM copied
|
var mbox, target int
|
||||||
|
err := DB.QueryRow(`SELECT COALESCE(MAX(mbox_done),0), COALESCE(MAX(target_done),0) FROM copied
|
||||||
WHERE account_id=? AND folder=? AND body_sha256=?`, accountID, candidateFolder, identity.BodySHA256).Scan(&mbox, &target)
|
WHERE account_id=? AND folder=? AND body_sha256=?`, accountID, candidateFolder, identity.BodySHA256).Scan(&mbox, &target)
|
||||||
} else {
|
if err != nil {
|
||||||
err = DB.QueryRow(`SELECT COALESCE(MAX(mbox_done),0), COALESCE(MAX(target_done),0) FROM copied
|
return CopyState{}, err
|
||||||
|
}
|
||||||
|
mboxDone |= mbox
|
||||||
|
targetDone |= target
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, alias := range uniqueAliases {
|
||||||
|
var mbox, target int
|
||||||
|
err := DB.QueryRow(`SELECT COALESCE(MAX(mbox_done),0), COALESCE(MAX(target_done),0) FROM copied
|
||||||
WHERE account_id=? AND folder=? AND message_id=? AND body_sha256=?`,
|
WHERE account_id=? AND folder=? AND message_id=? AND body_sha256=?`,
|
||||||
accountID, candidateFolder, identity.MessageID, identity.BodySHA256).Scan(&mbox, &target)
|
accountID, candidateFolder, alias, identity.BodySHA256).Scan(&mbox, &target)
|
||||||
|
if err != nil {
|
||||||
|
return CopyState{}, err
|
||||||
|
}
|
||||||
|
mboxDone |= mbox
|
||||||
|
targetDone |= target
|
||||||
}
|
}
|
||||||
if err != nil {
|
|
||||||
return CopyState{}, err
|
|
||||||
}
|
|
||||||
mboxDone |= mbox
|
|
||||||
targetDone |= target
|
|
||||||
}
|
}
|
||||||
if mboxDone != 0 || targetDone != 0 {
|
if mboxDone != 0 || targetDone != 0 {
|
||||||
return CopyState{MboxDone: mboxDone != 0, TargetDone: targetDone != 0}, nil
|
return CopyState{MboxDone: mboxDone != 0, TargetDone: targetDone != 0}, nil
|
||||||
}
|
}
|
||||||
// Once a Message-ID has precise archive identities, a different body with
|
// Once a Message-ID has precise archive identities, a different body with
|
||||||
// that same ID is new and must not be hidden by the old wildcard row.
|
// that same ID is new and must not be hidden by the old wildcard row.
|
||||||
if identity.MessageID != "" && identity.BodySHA256 != "" {
|
if len(uniqueAliases) != 0 && identity.BodySHA256 != "" {
|
||||||
var precise int
|
var precise int
|
||||||
for _, candidateFolder := range folders {
|
for _, candidateFolder := range folders {
|
||||||
var count int
|
for _, alias := range uniqueAliases {
|
||||||
if err := DB.QueryRow(`SELECT count(*) FROM copied WHERE account_id=? AND folder=? AND message_id=? AND body_sha256<>''`,
|
var count int
|
||||||
accountID, candidateFolder, identity.MessageID).Scan(&count); err != nil {
|
if err := DB.QueryRow(`SELECT count(*) FROM copied WHERE account_id=? AND folder=? AND message_id=? AND body_sha256<>''`,
|
||||||
return CopyState{}, err
|
accountID, candidateFolder, alias).Scan(&count); err != nil {
|
||||||
|
return CopyState{}, err
|
||||||
|
}
|
||||||
|
precise += count
|
||||||
}
|
}
|
||||||
precise += count
|
|
||||||
}
|
}
|
||||||
if precise > 0 {
|
if precise > 0 {
|
||||||
return CopyState{}, nil
|
return CopyState{}, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
aliases := []string{identity.MessageID, identity.LegacyMessageID}
|
for _, alias := range uniqueAliases {
|
||||||
if identity.MessageID == "" && identity.BodySHA256 != "" {
|
|
||||||
aliases = append(aliases, "sha256:"+identity.BodySHA256)
|
|
||||||
}
|
|
||||||
seen := map[string]bool{}
|
|
||||||
for _, alias := range aliases {
|
|
||||||
alias = strings.TrimSpace(alias)
|
|
||||||
if alias == "" || seen[alias] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
seen[alias] = true
|
|
||||||
for _, candidateFolder := range folders {
|
for _, candidateFolder := range folders {
|
||||||
var mbox, target int
|
var mbox, target int
|
||||||
err := DB.QueryRow(`SELECT COALESCE(MAX(mbox_done),0), COALESCE(MAX(target_done),0) FROM copied
|
err := DB.QueryRow(`SELECT COALESCE(MAX(mbox_done),0), COALESCE(MAX(target_done),0) FROM copied
|
||||||
|
|
@ -875,21 +947,25 @@ func ReplaceMboxIndex(accountID int64, folder string, entries []MboxIndexEntry,
|
||||||
if err := aliasRows.Close(); err != nil {
|
if err := aliasRows.Close(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
legacyIDsBySeq := map[int]string{}
|
type storedIndexIdentity struct {
|
||||||
|
messageID string
|
||||||
|
bodySHA256 string
|
||||||
|
}
|
||||||
|
storedIdentityBySeq := map[int]storedIndexIdentity{}
|
||||||
for _, candidateFolder := range folderAliases {
|
for _, candidateFolder := range folderAliases {
|
||||||
rows, err := tx.Query(`SELECT seq, message_id FROM mbox_index WHERE account_id=? AND folder=?`, accountID, candidateFolder)
|
rows, err := tx.Query(`SELECT seq, message_id, body_sha256 FROM mbox_index WHERE account_id=? AND folder=?`, accountID, candidateFolder)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var seq int
|
var seq int
|
||||||
var messageID string
|
var identity storedIndexIdentity
|
||||||
if err := rows.Scan(&seq, &messageID); err != nil {
|
if err := rows.Scan(&seq, &identity.messageID, &identity.bodySHA256); err != nil {
|
||||||
_ = rows.Close()
|
_ = rows.Close()
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if _, exists := legacyIDsBySeq[seq]; !exists || candidateFolder == folder {
|
if _, exists := storedIdentityBySeq[seq]; !exists || candidateFolder == folder {
|
||||||
legacyIDsBySeq[seq] = messageID
|
storedIdentityBySeq[seq] = identity
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if err := rows.Err(); err != nil {
|
if err := rows.Err(); err != nil {
|
||||||
|
|
@ -907,6 +983,15 @@ func ReplaceMboxIndex(accountID int64, folder string, entries []MboxIndexEntry,
|
||||||
entry.AccountID = accountID
|
entry.AccountID = accountID
|
||||||
entry.Folder = folder
|
entry.Folder = folder
|
||||||
entry.Seq = i
|
entry.Seq = i
|
||||||
|
storedIdentity := storedIdentityBySeq[i]
|
||||||
|
// The IMAP envelope and a later RFC-822 header parse can represent the
|
||||||
|
// same malformed Message-ID differently. If the canonical body at the
|
||||||
|
// same archive position is identical, retain the already committed ID so
|
||||||
|
// a rebuild is an exact metadata restore. Never carry it across a body
|
||||||
|
// change, because that could attach an old identity to different content.
|
||||||
|
if storedIdentity.bodySHA256 != "" && storedIdentity.bodySHA256 == entry.BodySHA256 {
|
||||||
|
entry.MessageID = storedIdentity.messageID
|
||||||
|
}
|
||||||
if entry.MessageID == "" && entry.BodySHA256 == "" {
|
if entry.MessageID == "" && entry.BodySHA256 == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
@ -921,7 +1006,7 @@ func ReplaceMboxIndex(accountID int64, folder string, entries []MboxIndexEntry,
|
||||||
// pipeline the target append happened before the mbox append, so recovered
|
// pipeline the target append happened before the mbox append, so recovered
|
||||||
// records are complete in both stages. Marking them prevents a later watch
|
// records are complete in both stages. Marking them prevents a later watch
|
||||||
// run from appending duplicates to either destination.
|
// run from appending duplicates to either destination.
|
||||||
state, found, err := copyStateForReindex(tx, entry.AccountID, folderAliases, entry, legacyIDsBySeq[i])
|
state, found, err := copyStateForReindex(tx, entry.AccountID, folderAliases, entry, storedIdentity.messageID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -31,10 +31,13 @@ type RawMessage struct {
|
||||||
// for byte-different messages. Messages without a Message-ID are identified by
|
// for byte-different messages. Messages without a Message-ID are identified by
|
||||||
// BodySHA256 alone. LegacyMessageID keeps the old raw-IMAP hash addressable
|
// BodySHA256 alone. LegacyMessageID keeps the old raw-IMAP hash addressable
|
||||||
// while existing databases are migrated without re-copying mail.
|
// while existing databases are migrated without re-copying mail.
|
||||||
|
// LegacyMessageIDs additionally retain historical Message-ID normalizations
|
||||||
|
// from both the IMAP envelope and the raw RFC-822 header.
|
||||||
type MessageIdentity struct {
|
type MessageIdentity struct {
|
||||||
MessageID string
|
MessageID string
|
||||||
BodySHA256 string
|
BodySHA256 string
|
||||||
LegacyMessageID string
|
LegacyMessageID string
|
||||||
|
LegacyMessageIDs []string
|
||||||
}
|
}
|
||||||
|
|
||||||
type MessageHeader struct {
|
type MessageHeader struct {
|
||||||
|
|
@ -489,19 +492,23 @@ func messageID(body []byte) string {
|
||||||
|
|
||||||
func identityForRawMessage(m RawMessage) MessageIdentity {
|
func identityForRawMessage(m RawMessage) MessageIdentity {
|
||||||
id := normalizeMessageID(m.MessageID)
|
id := normalizeMessageID(m.MessageID)
|
||||||
legacyID := id
|
legacyID := legacyNormalizeMessageID(m.MessageID)
|
||||||
|
aliases := []string{legacyID}
|
||||||
if strings.HasPrefix(strings.ToLower(id), "sha256:") {
|
if strings.HasPrefix(strings.ToLower(id), "sha256:") {
|
||||||
id = ""
|
id = ""
|
||||||
}
|
}
|
||||||
if id == "" {
|
if msg, err := mail.ReadMessage(bytes.NewReader(m.Body)); err == nil {
|
||||||
if msg, err := mail.ReadMessage(bytes.NewReader(m.Body)); err == nil {
|
headerID := msg.Header.Get("Message-ID")
|
||||||
id = normalizeMessageID(msg.Header.Get("Message-ID"))
|
aliases = append(aliases, normalizeMessageID(headerID), legacyNormalizeMessageID(headerID))
|
||||||
|
if id == "" {
|
||||||
|
id = normalizeMessageID(headerID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return MessageIdentity{
|
return MessageIdentity{
|
||||||
MessageID: id,
|
MessageID: id,
|
||||||
BodySHA256: bodySHA256(m.Body),
|
BodySHA256: bodySHA256(m.Body),
|
||||||
LegacyMessageID: legacyID,
|
LegacyMessageID: legacyID,
|
||||||
|
LegacyMessageIDs: aliases,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -525,6 +532,18 @@ func bodySHA256(raw []byte) string {
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizeMessageID(id string) string {
|
func normalizeMessageID(id string) string {
|
||||||
|
id = strings.TrimSpace(id)
|
||||||
|
if start := strings.IndexByte(id, '<'); start >= 0 {
|
||||||
|
if relativeEnd := strings.IndexByte(id[start+1:], '>'); relativeEnd >= 0 {
|
||||||
|
if candidate := strings.TrimSpace(id[start+1 : start+1+relativeEnd]); candidate != "" {
|
||||||
|
return candidate
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return strings.Trim(id, "<>")
|
||||||
|
}
|
||||||
|
|
||||||
|
func legacyNormalizeMessageID(id string) string {
|
||||||
return strings.Trim(strings.TrimSpace(id), "<>")
|
return strings.Trim(strings.TrimSpace(id), "<>")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -25,6 +25,56 @@ func TestMessageIDFallsBackToBodyHash(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestNormalizeMessageIDExtractsBracketedIDBeforeBrokenComment(t *testing.T) {
|
||||||
|
tests := map[string]string{
|
||||||
|
"<normal@example.com>": "normal@example.com",
|
||||||
|
"bracketless@example.com": "bracketless@example.com",
|
||||||
|
" <5486CB7400AF1FB7@mr001msb.fastweb.it> (added by postmaster) ": "5486CB7400AF1FB7@mr001msb.fastweb.it",
|
||||||
|
"comment <embedded@example.com> trailing": "embedded@example.com",
|
||||||
|
}
|
||||||
|
for input, want := range tests {
|
||||||
|
if got := normalizeMessageID(input); got != want {
|
||||||
|
t.Errorf("normalizeMessageID(%q)=%q, want %q", input, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIdentityKeepsOldMalformedHeaderNormalizationAsAlias(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
header string
|
||||||
|
clean string
|
||||||
|
legacy string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
header: `<476BDFDB019289BE@mail21.bluewin.ch> (added by postmaster@bluewin.ch)`,
|
||||||
|
clean: `476BDFDB019289BE@mail21.bluewin.ch`,
|
||||||
|
legacy: `476BDFDB019289BE@mail21.bluewin.ch> (added by postmaster@bluewin.ch)`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
header: `<22578196653698419217.BB8A0E8F48F3F290@dr-gold.de>+D271B1409C3C6028`,
|
||||||
|
clean: `22578196653698419217.BB8A0E8F48F3F290@dr-gold.de`,
|
||||||
|
legacy: `22578196653698419217.BB8A0E8F48F3F290@dr-gold.de>+D271B1409C3C6028`,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
body := []byte("Message-ID: " + tt.header + "\r\nSubject: malformed ID\r\n\r\nbody")
|
||||||
|
identity := identityForRawMessage(RawMessage{MessageID: messageID(body), Body: body})
|
||||||
|
if identity.MessageID != tt.clean {
|
||||||
|
t.Fatalf("MessageID=%q, want %q", identity.MessageID, tt.clean)
|
||||||
|
}
|
||||||
|
found := false
|
||||||
|
for _, alias := range identity.LegacyMessageIDs {
|
||||||
|
if alias == tt.legacy {
|
||||||
|
found = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatalf("legacy alias %q missing from %#v", tt.legacy, identity.LegacyMessageIDs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestBodySHA256IsStableAcrossMboxNormalization(t *testing.T) {
|
func TestBodySHA256IsStableAcrossMboxNormalization(t *testing.T) {
|
||||||
raw := []byte("From: a@example.com\r\nSubject: Test\r\n\r\nFirst\r\nFrom escaped\r\n\r\n")
|
raw := []byte("From: a@example.com\r\nSubject: Test\r\n\r\nFirst\r\nFrom escaped\r\n\r\n")
|
||||||
record := mboxRecord(RawMessage{Body: raw})
|
record := mboxRecord(RawMessage{Body: raw})
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
package backend
|
package backend
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bufio"
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
@ -290,17 +291,55 @@ func reindexAccountArchive(account Account) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
func reindexPlainMbox(path string) ([]MboxIndexEntry, error) {
|
func reindexPlainMbox(path string) ([]MboxIndexEntry, error) {
|
||||||
b, err := os.ReadFile(path)
|
f, err := os.Open(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
defer f.Close()
|
||||||
|
reader := bufio.NewReaderSize(f, 256*1024)
|
||||||
var out []MboxIndexEntry
|
var out []MboxIndexEntry
|
||||||
for _, part := range splitMboxRecordsWithOffsets(b) {
|
var record bytes.Buffer
|
||||||
entry := indexEntryFromMboxRecord(part.Message, part.Offset, int64(part.Length), 0, int64(part.Length))
|
var offset int64
|
||||||
|
var messageStart int64 = -1
|
||||||
|
appendRecord := func(end int64) {
|
||||||
|
if messageStart < 0 || end <= messageStart {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
message := unescapeMboxMessage(record.Bytes())
|
||||||
|
if len(message) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
entry := indexEntryFromMboxRecord(message, messageStart, end-messageStart, 0, end-messageStart)
|
||||||
if entry.MessageID != "" || entry.BodySHA256 != "" {
|
if entry.MessageID != "" || entry.BodySHA256 != "" {
|
||||||
out = append(out, entry)
|
out = append(out, entry)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
for {
|
||||||
|
lineStart := offset
|
||||||
|
line, readErr := reader.ReadBytes('\n')
|
||||||
|
offset += int64(len(line))
|
||||||
|
content := line
|
||||||
|
if len(content) > 0 && content[len(content)-1] == '\n' {
|
||||||
|
content = content[:len(content)-1]
|
||||||
|
}
|
||||||
|
if len(content) > 0 && content[len(content)-1] == '\r' {
|
||||||
|
content = content[:len(content)-1]
|
||||||
|
}
|
||||||
|
if bytes.HasPrefix(content, []byte("From ")) {
|
||||||
|
appendRecord(lineStart)
|
||||||
|
record = bytes.Buffer{}
|
||||||
|
messageStart = offset
|
||||||
|
} else if messageStart >= 0 && len(line) > 0 {
|
||||||
|
_, _ = record.Write(line)
|
||||||
|
}
|
||||||
|
if readErr != nil {
|
||||||
|
if readErr != io.EOF {
|
||||||
|
return nil, readErr
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
appendRecord(offset)
|
||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -3,8 +3,10 @@ package backend
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
@ -12,6 +14,39 @@ import (
|
||||||
"github.com/klauspost/compress/zstd"
|
"github.com/klauspost/compress/zstd"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestReindexPlainMboxStreamingMatchesReferenceParser(t *testing.T) {
|
||||||
|
var mbox bytes.Buffer
|
||||||
|
for i := 0; i < 257; i++ {
|
||||||
|
body := strings.Repeat(fmt.Sprintf("line-%03d From inside body\r\n", i), i%31+1)
|
||||||
|
mbox.Write(mboxRecord(RawMessage{
|
||||||
|
MessageID: fmt.Sprintf("stream-%03d@example.com", i),
|
||||||
|
Body: []byte(fmt.Sprintf(
|
||||||
|
"Message-ID: <stream-%03d@example.com>\r\nSubject: Stream %03d\r\n\r\n%s",
|
||||||
|
i, i, body,
|
||||||
|
)),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
path := filepath.Join(t.TempDir(), "streaming.mbox")
|
||||||
|
if err := os.WriteFile(path, mbox.Bytes(), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := reindexPlainMbox(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
parts := splitMboxRecordsWithOffsets(mbox.Bytes())
|
||||||
|
if len(got) != len(parts) {
|
||||||
|
t.Fatalf("streaming entries=%d, reference parts=%d", len(got), len(parts))
|
||||||
|
}
|
||||||
|
for i, part := range parts {
|
||||||
|
want := indexEntryFromMboxRecord(part.Message, part.Offset, int64(part.Length), 0, int64(part.Length))
|
||||||
|
if got[i] != want {
|
||||||
|
t.Fatalf("entry %d differs:\n got %#v\n want %#v", i, got[i], want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestZstdMboxRoundTripAndIndexRead(t *testing.T) {
|
func TestZstdMboxRoundTripAndIndexRead(t *testing.T) {
|
||||||
oldCfg, oldDB := Cfg, DB
|
oldCfg, oldDB := Cfg, DB
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
|
|
@ -325,6 +360,14 @@ func TestConnectDBConfiguresBusyTimeoutAndCopyStages(t *testing.T) {
|
||||||
Cfg, DB = oldCfg, oldDB
|
Cfg, DB = oldCfg, oldDB
|
||||||
})
|
})
|
||||||
Cfg = Config{DBPath: filepath.Join(root, "mail-graveyard.db"), MboxRoot: filepath.Join(root, "backup")}
|
Cfg = Config{DBPath: filepath.Join(root, "mail-graveyard.db"), MboxRoot: filepath.Join(root, "backup")}
|
||||||
|
backupDB := filepath.Join(root, "PRE-existing-copy.db")
|
||||||
|
if err := os.WriteFile(backupDB, []byte("diagnostic copy"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
nonDB := filepath.Join(root, "keep-mode.txt")
|
||||||
|
if err := os.WriteFile(nonDB, []byte("not a database"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
if err := ConnectDB(true); err != nil {
|
if err := ConnectDB(true); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
@ -335,6 +378,24 @@ func TestConnectDBConfiguresBusyTimeoutAndCopyStages(t *testing.T) {
|
||||||
if timeout != 10000 {
|
if timeout != 10000 {
|
||||||
t.Fatalf("busy_timeout=%d, want 10000", timeout)
|
t.Fatalf("busy_timeout=%d, want 10000", timeout)
|
||||||
}
|
}
|
||||||
|
if runtime.GOOS != "windows" {
|
||||||
|
for _, path := range []string{Cfg.DBPath, Cfg.DBPath + "-wal", Cfg.DBPath + "-shm", backupDB} {
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if permissions := info.Mode().Perm(); permissions != 0o600 {
|
||||||
|
t.Fatalf("SQLite file %s permissions=%#o, want 0600", path, permissions)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
info, err := os.Stat(nonDB)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if permissions := info.Mode().Perm(); permissions != 0o644 {
|
||||||
|
t.Fatalf("non-DB file permissions changed to %#o", permissions)
|
||||||
|
}
|
||||||
|
}
|
||||||
if err := SaveAccount(Account{
|
if err := SaveAccount(Account{
|
||||||
Name: "stage-account", SrcHost: "source.example", SrcPort: 993, SrcSecurity: "tls", SrcUser: "source@example.com", SrcPass: "x",
|
Name: "stage-account", SrcHost: "source.example", SrcPort: 993, SrcSecurity: "tls", SrcUser: "source@example.com", SrcPass: "x",
|
||||||
DstHost: "target.example", DstPort: 993, DstSecurity: "tls", DstUser: "target@example.com", DstPass: "x", Active: true,
|
DstHost: "target.example", DstPort: 993, DstSecurity: "tls", DstUser: "target@example.com", DstPass: "x", Active: true,
|
||||||
|
|
@ -379,6 +440,106 @@ func TestConnectDBConfiguresBusyTimeoutAndCopyStages(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestReplaceMboxIndexPreservesStoredMessageIDOnlyForSameBody(t *testing.T) {
|
||||||
|
oldCfg, oldDB := Cfg, DB
|
||||||
|
root := t.TempDir()
|
||||||
|
t.Cleanup(func() {
|
||||||
|
if DB != nil {
|
||||||
|
_ = DB.Close()
|
||||||
|
}
|
||||||
|
Cfg, DB = oldCfg, oldDB
|
||||||
|
})
|
||||||
|
Cfg = Config{DBPath: filepath.Join(root, "mail-graveyard.db"), MboxRoot: filepath.Join(root, "backup")}
|
||||||
|
if err := ConnectDB(true); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := SaveAccount(Account{
|
||||||
|
Name: "identity-restore", SrcHost: "source.example", SrcPort: 993, SrcSecurity: "tls", SrcUser: "source@example.com", SrcPass: "x",
|
||||||
|
DstHost: "target.example", DstPort: 993, DstSecurity: "tls", DstUser: "target@example.com", DstPass: "x", Active: true,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
account, err := GetAccount("identity-restore")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
stableHash := bodySHA256([]byte("same archived body"))
|
||||||
|
if err := ReplaceMboxIndex(account.ID, "INBOX", []MboxIndexEntry{{
|
||||||
|
MessageID: "stored@example.com> (added by postmaster)", BodySHA256: stableHash, FrameLen: 10, InnerLen: 10,
|
||||||
|
}}, 10); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := ReplaceMboxIndex(account.ID, "INBOX", []MboxIndexEntry{{
|
||||||
|
MessageID: "parsed@example.com", BodySHA256: stableHash, FrameLen: 10, InnerLen: 10,
|
||||||
|
}}, 10); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
entry, err := GetMboxIndex(account.ID, "INBOX", 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if entry.MessageID != "stored@example.com> (added by postmaster)" {
|
||||||
|
t.Fatalf("same body lost stored Message-ID: %q", entry.MessageID)
|
||||||
|
}
|
||||||
|
|
||||||
|
changedHash := bodySHA256([]byte("different archived body"))
|
||||||
|
if err := ReplaceMboxIndex(account.ID, "INBOX", []MboxIndexEntry{{
|
||||||
|
MessageID: "new-content@example.com", BodySHA256: changedHash, FrameLen: 11, InnerLen: 11,
|
||||||
|
}}, 11); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
entry, err = GetMboxIndex(account.ID, "INBOX", 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if entry.MessageID != "new-content@example.com" {
|
||||||
|
t.Fatalf("different body inherited stale Message-ID: %q", entry.MessageID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPreciseLegacyMessageIDAliasesPreventLiveRecopy(t *testing.T) {
|
||||||
|
account, _, restore := setupSnapshotTest(t, "none")
|
||||||
|
defer restore()
|
||||||
|
tests := []struct {
|
||||||
|
header string
|
||||||
|
legacy string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
header: `<476BDFDB019289BE@mail21.bluewin.ch> (added by postmaster@bluewin.ch)`,
|
||||||
|
legacy: `476BDFDB019289BE@mail21.bluewin.ch> (added by postmaster@bluewin.ch)`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
header: `<22578196653698419217.BB8A0E8F48F3F290@dr-gold.de>+D271B1409C3C6028`,
|
||||||
|
legacy: `22578196653698419217.BB8A0E8F48F3F290@dr-gold.de>+D271B1409C3C6028`,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for i, tt := range tests {
|
||||||
|
folder := fmt.Sprintf("INBOX.%d", i)
|
||||||
|
body := []byte("Message-ID: " + tt.header + "\r\nSubject: malformed ID\r\n\r\noriginal body")
|
||||||
|
identity := identityForRawMessage(RawMessage{MessageID: messageID(body), Body: body})
|
||||||
|
if err := MarkIdentityCopied(account.ID, folder, MessageIdentity{MessageID: tt.legacy, BodySHA256: identity.BodySHA256}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
state, err := GetCopyIdentityState(account.ID, folder, identity)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !state.MboxDone || !state.TargetDone {
|
||||||
|
t.Fatalf("fixture %d precise legacy alias was not recognized: %#v", i, state)
|
||||||
|
}
|
||||||
|
|
||||||
|
changedBody := []byte("Message-ID: " + tt.header + "\r\nSubject: malformed ID\r\n\r\ndifferent body")
|
||||||
|
changedIdentity := identityForRawMessage(RawMessage{MessageID: messageID(changedBody), Body: changedBody})
|
||||||
|
state, err = GetCopyIdentityState(account.ID, folder, changedIdentity)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if state.MboxDone || state.TargetDone {
|
||||||
|
t.Fatalf("fixture %d legacy alias hid byte-different mail: %#v", i, state)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestIdentitySchemaUpgradePreservesLegacyAliases(t *testing.T) {
|
func TestIdentitySchemaUpgradePreservesLegacyAliases(t *testing.T) {
|
||||||
oldCfg, oldDB := Cfg, DB
|
oldCfg, oldDB := Cfg, DB
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
|
|
|
||||||
711
backend/14-snapshot.go
Normal file
711
backend/14-snapshot.go
Normal file
|
|
@ -0,0 +1,711 @@
|
||||||
|
package backend
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"database/sql"
|
||||||
|
"encoding/binary"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const snapshotFormatVersion = 1
|
||||||
|
|
||||||
|
type SnapshotManifest struct {
|
||||||
|
FormatVersion int `json:"format_version"`
|
||||||
|
CreatedAt string `json:"created_at"`
|
||||||
|
DatabasePath string `json:"database_path"`
|
||||||
|
DatabaseSHA256 string `json:"database_sha256"`
|
||||||
|
IndexSHA256 string `json:"index_sha256"`
|
||||||
|
IndexRecords int64 `json:"index_records"`
|
||||||
|
PreciseCopied int64 `json:"precise_copied"`
|
||||||
|
PendingMbox int64 `json:"pending_mbox"`
|
||||||
|
PendingTarget int64 `json:"pending_target"`
|
||||||
|
InvalidStageOrder int64 `json:"invalid_stage_order"`
|
||||||
|
MissingArchiveRecords int64 `json:"missing_archive_records"`
|
||||||
|
Files []SnapshotFile `json:"files"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type SnapshotFile struct {
|
||||||
|
Path string `json:"path"`
|
||||||
|
Bytes int64 `json:"bytes"`
|
||||||
|
SHA256 string `json:"sha256"`
|
||||||
|
AccountID int64 `json:"account_id"`
|
||||||
|
Account string `json:"account"`
|
||||||
|
Folder string `json:"folder"`
|
||||||
|
IndexRecords int64 `json:"index_records"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type snapshotWatermark struct {
|
||||||
|
AccountID int64
|
||||||
|
Account string
|
||||||
|
MboxDir string
|
||||||
|
Folder string
|
||||||
|
Bytes int64
|
||||||
|
IndexRecords int64
|
||||||
|
SourcePath string
|
||||||
|
RelativePath string
|
||||||
|
}
|
||||||
|
|
||||||
|
// CreateSnapshot creates a self-consistent DB + mbox-prefix snapshot without
|
||||||
|
// stopping append-only archive writers. The DB snapshot is taken first; every
|
||||||
|
// copied mbox prefix is then bounded exclusively by offsets from that snapshot.
|
||||||
|
func CreateSnapshot(target string) error {
|
||||||
|
return createSnapshot(target, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func createSnapshot(target string, afterDatabaseSnapshot func() error) (retErr error) {
|
||||||
|
if DB == nil {
|
||||||
|
return fmt.Errorf("snapshot: database is not connected")
|
||||||
|
}
|
||||||
|
target = strings.TrimSpace(target)
|
||||||
|
if target == "" {
|
||||||
|
return fmt.Errorf("snapshot: target directory is empty")
|
||||||
|
}
|
||||||
|
targetAbs, err := filepath.Abs(target)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("snapshot target: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(targetAbs); err == nil {
|
||||||
|
return fmt.Errorf("snapshot target already exists: %s", targetAbs)
|
||||||
|
} else if !os.IsNotExist(err) {
|
||||||
|
return fmt.Errorf("snapshot target stat: %w", err)
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(targetAbs), 0o700); err != nil {
|
||||||
|
return fmt.Errorf("snapshot target parent: %w", err)
|
||||||
|
}
|
||||||
|
partial := fmt.Sprintf("%s.partial-%d-%d", targetAbs, os.Getpid(), time.Now().UnixNano())
|
||||||
|
if err := os.Mkdir(partial, 0o700); err != nil {
|
||||||
|
return fmt.Errorf("snapshot staging directory: %w", err)
|
||||||
|
}
|
||||||
|
complete := false
|
||||||
|
defer func() {
|
||||||
|
if !complete {
|
||||||
|
log.Printf("snapshot incomplete; staging data retained at %s", partial)
|
||||||
|
if retErr != nil {
|
||||||
|
retErr = fmt.Errorf("%w (incomplete snapshot retained at %s)", retErr, partial)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
databaseRel := filepath.ToSlash(filepath.Join("data", "mail-graveyard.db"))
|
||||||
|
databasePath := filepath.Join(partial, filepath.FromSlash(databaseRel))
|
||||||
|
if err := os.MkdirAll(filepath.Dir(databasePath), 0o700); err != nil {
|
||||||
|
return fmt.Errorf("snapshot database directory: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := DB.Exec(`VACUUM INTO ?`, databasePath); err != nil {
|
||||||
|
return fmt.Errorf("snapshot database VACUUM INTO: %w", err)
|
||||||
|
}
|
||||||
|
if err := chmodPrivateFile(databasePath); err != nil {
|
||||||
|
return fmt.Errorf("snapshot database permissions: %w", err)
|
||||||
|
}
|
||||||
|
if err := syncFile(databasePath); err != nil {
|
||||||
|
return fmt.Errorf("snapshot database sync: %w", err)
|
||||||
|
}
|
||||||
|
if afterDatabaseSnapshot != nil {
|
||||||
|
if err := afterDatabaseSnapshot(); err != nil {
|
||||||
|
return fmt.Errorf("snapshot test hook: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshotDB, err := openSnapshotDatabase(databasePath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
watermarks, err := collectSnapshotWatermarks(snapshotDB, Cfg.MboxRoot)
|
||||||
|
if err != nil {
|
||||||
|
_ = snapshotDB.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
stats, err := snapshotDatabaseStats(snapshotDB)
|
||||||
|
if err == nil {
|
||||||
|
stats.IndexSHA256, err = indexDigest(snapshotDB)
|
||||||
|
}
|
||||||
|
if closeErr := snapshotDB.Close(); err == nil && closeErr != nil {
|
||||||
|
err = closeErr
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if stats.MissingArchiveRecords != 0 {
|
||||||
|
return fmt.Errorf("snapshot refused: %d mbox_done identities have no archive index", stats.MissingArchiveRecords)
|
||||||
|
}
|
||||||
|
if stats.InvalidStageOrder != 0 {
|
||||||
|
return fmt.Errorf("snapshot refused: %d precise identities have target_done=1 before mbox_done", stats.InvalidStageOrder)
|
||||||
|
}
|
||||||
|
var watermarkRecords int64
|
||||||
|
for _, watermark := range watermarks {
|
||||||
|
watermarkRecords += watermark.IndexRecords
|
||||||
|
}
|
||||||
|
if watermarkRecords != stats.IndexRecords {
|
||||||
|
return fmt.Errorf("snapshot refused: indexed records=%d but mapped mbox records=%d", stats.IndexRecords, watermarkRecords)
|
||||||
|
}
|
||||||
|
|
||||||
|
manifest := SnapshotManifest{
|
||||||
|
FormatVersion: snapshotFormatVersion,
|
||||||
|
CreatedAt: time.Now().UTC().Format(time.RFC3339Nano),
|
||||||
|
DatabasePath: databaseRel,
|
||||||
|
IndexSHA256: stats.IndexSHA256,
|
||||||
|
IndexRecords: stats.IndexRecords,
|
||||||
|
PreciseCopied: stats.PreciseCopied,
|
||||||
|
PendingMbox: stats.PendingMbox,
|
||||||
|
PendingTarget: stats.PendingTarget,
|
||||||
|
InvalidStageOrder: stats.InvalidStageOrder,
|
||||||
|
MissingArchiveRecords: stats.MissingArchiveRecords,
|
||||||
|
Files: make([]SnapshotFile, 0, len(watermarks)),
|
||||||
|
}
|
||||||
|
manifest.DatabaseSHA256, err = hashFile(databasePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("snapshot database hash: %w", err)
|
||||||
|
}
|
||||||
|
for _, watermark := range watermarks {
|
||||||
|
destinationRel := filepath.ToSlash(filepath.Join("backup", watermark.RelativePath))
|
||||||
|
destination := filepath.Join(partial, filepath.FromSlash(destinationRel))
|
||||||
|
hash, err := copyFilePrefix(watermark.SourcePath, destination, watermark.Bytes)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("snapshot mbox %s: %w", destinationRel, err)
|
||||||
|
}
|
||||||
|
manifest.Files = append(manifest.Files, SnapshotFile{
|
||||||
|
Path: destinationRel,
|
||||||
|
Bytes: watermark.Bytes,
|
||||||
|
SHA256: hash,
|
||||||
|
AccountID: watermark.AccountID,
|
||||||
|
Account: watermark.Account,
|
||||||
|
Folder: watermark.Folder,
|
||||||
|
IndexRecords: watermark.IndexRecords,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
sortSnapshotFiles(manifest.Files)
|
||||||
|
manifestPath := filepath.Join(partial, "snapshot-manifest.json")
|
||||||
|
manifestJSON, err := json.MarshalIndent(manifest, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("snapshot manifest encode: %w", err)
|
||||||
|
}
|
||||||
|
manifestJSON = append(manifestJSON, '\n')
|
||||||
|
if err := os.WriteFile(manifestPath, manifestJSON, 0o600); err != nil {
|
||||||
|
return fmt.Errorf("snapshot manifest write: %w", err)
|
||||||
|
}
|
||||||
|
if err := syncFile(manifestPath); err != nil {
|
||||||
|
return fmt.Errorf("snapshot manifest sync: %w", err)
|
||||||
|
}
|
||||||
|
if err := os.Rename(partial, targetAbs); err != nil {
|
||||||
|
return fmt.Errorf("snapshot finalize: %w", err)
|
||||||
|
}
|
||||||
|
complete = true
|
||||||
|
log.Printf("snapshot complete target=%s files=%d bytes=%d index_records=%d precise_copied=%d pending_mbox=%d pending_target=%d",
|
||||||
|
targetAbs, len(manifest.Files), snapshotBytes(manifest.Files), manifest.IndexRecords, manifest.PreciseCopied, manifest.PendingMbox, manifest.PendingTarget)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type snapshotStats struct {
|
||||||
|
IndexRecords int64
|
||||||
|
PreciseCopied int64
|
||||||
|
PendingMbox int64
|
||||||
|
PendingTarget int64
|
||||||
|
MissingArchiveRecords int64
|
||||||
|
InvalidStageOrder int64
|
||||||
|
IndexSHA256 string
|
||||||
|
}
|
||||||
|
|
||||||
|
func snapshotDatabaseStats(db *sql.DB) (snapshotStats, error) {
|
||||||
|
var stats snapshotStats
|
||||||
|
queries := []struct {
|
||||||
|
query string
|
||||||
|
dest *int64
|
||||||
|
}{
|
||||||
|
{`SELECT count(*) FROM mbox_index`, &stats.IndexRecords},
|
||||||
|
{`SELECT count(*) FROM copied WHERE body_sha256<>''`, &stats.PreciseCopied},
|
||||||
|
{`SELECT count(*) FROM copied WHERE body_sha256<>'' AND mbox_done=0`, &stats.PendingMbox},
|
||||||
|
{`SELECT count(*) FROM copied WHERE body_sha256<>'' AND target_done=0`, &stats.PendingTarget},
|
||||||
|
{`SELECT count(*) FROM copied WHERE body_sha256<>'' AND target_done=1 AND mbox_done=0`, &stats.InvalidStageOrder},
|
||||||
|
{`SELECT count(*) FROM copied c WHERE c.body_sha256<>'' AND c.mbox_done=1 AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM mbox_index i
|
||||||
|
WHERE i.account_id=c.account_id AND i.folder=c.folder
|
||||||
|
AND i.message_id=c.message_id AND i.body_sha256=c.body_sha256
|
||||||
|
)`, &stats.MissingArchiveRecords},
|
||||||
|
}
|
||||||
|
for _, item := range queries {
|
||||||
|
if err := db.QueryRow(item.query).Scan(item.dest); err != nil {
|
||||||
|
return snapshotStats{}, fmt.Errorf("snapshot database statistics: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return stats, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func indexDigest(db *sql.DB) (string, error) {
|
||||||
|
rows, err := db.Query(`SELECT account_id, folder, seq, message_id, body_sha256
|
||||||
|
FROM mbox_index ORDER BY account_id, folder, seq`)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("snapshot index digest query: %w", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
hash := sha256.New()
|
||||||
|
for rows.Next() {
|
||||||
|
var accountID, seq int64
|
||||||
|
var folder, messageID, bodyHash string
|
||||||
|
if err := rows.Scan(&accountID, &folder, &seq, &messageID, &bodyHash); err != nil {
|
||||||
|
return "", fmt.Errorf("snapshot index digest row: %w", err)
|
||||||
|
}
|
||||||
|
for _, value := range []int64{accountID, seq} {
|
||||||
|
if err := binary.Write(hash, binary.LittleEndian, value); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, value := range []string{folder, messageID, bodyHash} {
|
||||||
|
if err := binary.Write(hash, binary.LittleEndian, uint64(len(value))); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if _, err := io.WriteString(hash, value); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return "", fmt.Errorf("snapshot index digest: %w", err)
|
||||||
|
}
|
||||||
|
return hex.EncodeToString(hash.Sum(nil)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func collectSnapshotWatermarks(db *sql.DB, mboxRoot string) ([]snapshotWatermark, error) {
|
||||||
|
rootAbs, err := filepath.Abs(mboxRoot)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("snapshot mbox root: %w", err)
|
||||||
|
}
|
||||||
|
rows, err := db.Query(`SELECT i.account_id, a.name, a.mbox_dir, i.folder,
|
||||||
|
MAX(i.file_offset+i.frame_len), count(*)
|
||||||
|
FROM mbox_index i JOIN accounts a ON a.id=i.account_id
|
||||||
|
GROUP BY i.account_id, a.name, a.mbox_dir, i.folder
|
||||||
|
ORDER BY lower(a.name), a.name, i.folder`)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("snapshot watermarks: %w", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var out []snapshotWatermark
|
||||||
|
seenPaths := map[string]bool{}
|
||||||
|
archivePaths := map[string]map[string]string{}
|
||||||
|
for rows.Next() {
|
||||||
|
var watermark snapshotWatermark
|
||||||
|
if err := rows.Scan(&watermark.AccountID, &watermark.Account, &watermark.MboxDir, &watermark.Folder, &watermark.Bytes, &watermark.IndexRecords); err != nil {
|
||||||
|
return nil, fmt.Errorf("snapshot watermark row: %w", err)
|
||||||
|
}
|
||||||
|
watermark.MboxDir = strings.TrimSpace(watermark.MboxDir)
|
||||||
|
if watermark.MboxDir == "" {
|
||||||
|
return nil, fmt.Errorf("snapshot account %s has no mbox_dir", watermark.Account)
|
||||||
|
}
|
||||||
|
paths, ok := archivePaths[watermark.MboxDir]
|
||||||
|
if !ok {
|
||||||
|
paths = map[string]string{}
|
||||||
|
for _, path := range archiveMboxPaths(filepath.Join(rootAbs, watermark.MboxDir)) {
|
||||||
|
paths[folderNameFromMboxPath(path)] = path
|
||||||
|
}
|
||||||
|
archivePaths[watermark.MboxDir] = paths
|
||||||
|
}
|
||||||
|
watermark.SourcePath = paths[watermark.Folder]
|
||||||
|
if watermark.SourcePath == "" {
|
||||||
|
return nil, fmt.Errorf("snapshot indexed mbox missing for account=%s folder=%q", watermark.Account, watermark.Folder)
|
||||||
|
}
|
||||||
|
pathAbs, err := filepath.Abs(watermark.SourcePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("snapshot mbox path: %w", err)
|
||||||
|
}
|
||||||
|
relative, err := filepath.Rel(rootAbs, pathAbs)
|
||||||
|
if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(os.PathSeparator)) || filepath.IsAbs(relative) {
|
||||||
|
return nil, fmt.Errorf("snapshot mbox escapes configured root: %s", pathAbs)
|
||||||
|
}
|
||||||
|
watermark.SourcePath = pathAbs
|
||||||
|
watermark.RelativePath = relative
|
||||||
|
pathKey := snapshotPathKey(relative)
|
||||||
|
if seenPaths[pathKey] {
|
||||||
|
return nil, fmt.Errorf("snapshot multiple index groups resolve to the same mbox: %s", relative)
|
||||||
|
}
|
||||||
|
seenPaths[pathKey] = true
|
||||||
|
info, err := os.Stat(pathAbs)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("snapshot mbox stat %s: %w", pathAbs, err)
|
||||||
|
}
|
||||||
|
if !info.Mode().IsRegular() {
|
||||||
|
return nil, fmt.Errorf("snapshot mbox is not a regular file: %s", pathAbs)
|
||||||
|
}
|
||||||
|
if watermark.Bytes <= 0 || info.Size() < watermark.Bytes {
|
||||||
|
return nil, fmt.Errorf("snapshot invalid watermark account=%s folder=%q boundary=%d size=%d",
|
||||||
|
watermark.Account, watermark.Folder, watermark.Bytes, info.Size())
|
||||||
|
}
|
||||||
|
out = append(out, watermark)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, fmt.Errorf("snapshot watermarks: %w", err)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyFilePrefix(source, destination string, length int64) (string, error) {
|
||||||
|
if length < 0 {
|
||||||
|
return "", fmt.Errorf("negative prefix length %d", length)
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(destination), 0o700); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
src, err := os.Open(source)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer src.Close()
|
||||||
|
dst, err := os.OpenFile(destination, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
hash := sha256.New()
|
||||||
|
written, copyErr := io.CopyN(io.MultiWriter(dst, hash), src, length)
|
||||||
|
if copyErr == nil && written != length {
|
||||||
|
copyErr = io.ErrShortWrite
|
||||||
|
}
|
||||||
|
if copyErr == nil {
|
||||||
|
copyErr = dst.Sync()
|
||||||
|
}
|
||||||
|
closeErr := dst.Close()
|
||||||
|
if copyErr != nil {
|
||||||
|
return "", copyErr
|
||||||
|
}
|
||||||
|
if closeErr != nil {
|
||||||
|
return "", closeErr
|
||||||
|
}
|
||||||
|
return hex.EncodeToString(hash.Sum(nil)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func openSnapshotDatabase(path string) (*sql.DB, error) {
|
||||||
|
db, err := sql.Open("sqlite", path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("open snapshot database: %w", err)
|
||||||
|
}
|
||||||
|
db.SetMaxOpenConns(1)
|
||||||
|
if _, err := db.Exec(`PRAGMA query_only=ON; PRAGMA foreign_keys=ON;`); err != nil {
|
||||||
|
_ = db.Close()
|
||||||
|
return nil, fmt.Errorf("open snapshot database read-only: %w", err)
|
||||||
|
}
|
||||||
|
return db, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// VerifySnapshot validates hashes and DB/file boundaries, then copies only the
|
||||||
|
// small DB into a temporary restore workspace and performs a full reindex there.
|
||||||
|
// The snapshot mbox files and any live database remain untouched.
|
||||||
|
func VerifySnapshot(root string) error {
|
||||||
|
root = strings.TrimSpace(root)
|
||||||
|
if root == "" {
|
||||||
|
return fmt.Errorf("verify snapshot: directory is empty")
|
||||||
|
}
|
||||||
|
rootAbs, err := filepath.Abs(root)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("verify snapshot path: %w", err)
|
||||||
|
}
|
||||||
|
manifestBytes, err := os.ReadFile(filepath.Join(rootAbs, "snapshot-manifest.json"))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("verify snapshot manifest: %w", err)
|
||||||
|
}
|
||||||
|
var manifest SnapshotManifest
|
||||||
|
if err := json.Unmarshal(manifestBytes, &manifest); err != nil {
|
||||||
|
return fmt.Errorf("verify snapshot manifest decode: %w", err)
|
||||||
|
}
|
||||||
|
if manifest.FormatVersion != snapshotFormatVersion {
|
||||||
|
return fmt.Errorf("verify snapshot format=%d, supported=%d", manifest.FormatVersion, snapshotFormatVersion)
|
||||||
|
}
|
||||||
|
databasePath, err := pathInsideSnapshot(rootAbs, manifest.DatabasePath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if runtime.GOOS != "windows" {
|
||||||
|
databaseInfo, err := os.Stat(databasePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("verify snapshot database permissions: %w", err)
|
||||||
|
}
|
||||||
|
if permissions := databaseInfo.Mode().Perm(); permissions&0o077 != 0 {
|
||||||
|
return fmt.Errorf("verify snapshot database permissions=%#o, want no group/other access", permissions)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
databaseHash, err := hashFile(databasePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("verify snapshot database hash: %w", err)
|
||||||
|
}
|
||||||
|
if databaseHash != manifest.DatabaseSHA256 {
|
||||||
|
return fmt.Errorf("verify snapshot database hash mismatch: got %s want %s", databaseHash, manifest.DatabaseSHA256)
|
||||||
|
}
|
||||||
|
for _, file := range manifest.Files {
|
||||||
|
path, err := pathInsideSnapshot(rootAbs, file.Path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("verify snapshot file %s: %w", file.Path, err)
|
||||||
|
}
|
||||||
|
if info.Size() != file.Bytes {
|
||||||
|
return fmt.Errorf("verify snapshot file size %s: got %d want %d", file.Path, info.Size(), file.Bytes)
|
||||||
|
}
|
||||||
|
hash, err := hashFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("verify snapshot file hash %s: %w", file.Path, err)
|
||||||
|
}
|
||||||
|
if hash != file.SHA256 {
|
||||||
|
return fmt.Errorf("verify snapshot file hash mismatch %s: got %s want %s", file.Path, hash, file.SHA256)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshotDB, err := openSnapshotDatabase(databasePath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var integrity string
|
||||||
|
if err := snapshotDB.QueryRow(`PRAGMA integrity_check`).Scan(&integrity); err != nil {
|
||||||
|
_ = snapshotDB.Close()
|
||||||
|
return fmt.Errorf("verify snapshot integrity: %w", err)
|
||||||
|
}
|
||||||
|
if integrity != "ok" {
|
||||||
|
_ = snapshotDB.Close()
|
||||||
|
return fmt.Errorf("verify snapshot integrity: %s", integrity)
|
||||||
|
}
|
||||||
|
stats, err := snapshotDatabaseStats(snapshotDB)
|
||||||
|
if err == nil {
|
||||||
|
stats.IndexSHA256, err = indexDigest(snapshotDB)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
_ = snapshotDB.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
watermarks, err := collectSnapshotWatermarks(snapshotDB, filepath.Join(rootAbs, "backup"))
|
||||||
|
if closeErr := snapshotDB.Close(); err == nil && closeErr != nil {
|
||||||
|
err = closeErr
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if stats.IndexRecords != manifest.IndexRecords || stats.PreciseCopied != manifest.PreciseCopied ||
|
||||||
|
stats.PendingMbox != manifest.PendingMbox || stats.PendingTarget != manifest.PendingTarget ||
|
||||||
|
stats.InvalidStageOrder != manifest.InvalidStageOrder || stats.MissingArchiveRecords != manifest.MissingArchiveRecords ||
|
||||||
|
stats.IndexSHA256 != manifest.IndexSHA256 {
|
||||||
|
return fmt.Errorf("verify snapshot manifest/database statistics differ")
|
||||||
|
}
|
||||||
|
if stats.MissingArchiveRecords != 0 {
|
||||||
|
return fmt.Errorf("verify snapshot has %d mbox_done identities without archive index", stats.MissingArchiveRecords)
|
||||||
|
}
|
||||||
|
if stats.InvalidStageOrder != 0 {
|
||||||
|
return fmt.Errorf("verify snapshot has %d precise identities with invalid stage order", stats.InvalidStageOrder)
|
||||||
|
}
|
||||||
|
if err := compareManifestWatermarks(manifest.Files, watermarks); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := verifySnapshotReindex(rootAbs, databasePath, manifest.IndexRecords, manifest.IndexSHA256); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
log.Printf("snapshot verified root=%s files=%d bytes=%d index_records=%d precise_copied=%d pending_mbox=%d pending_target=%d",
|
||||||
|
rootAbs, len(manifest.Files), snapshotBytes(manifest.Files), manifest.IndexRecords, manifest.PreciseCopied, manifest.PendingMbox, manifest.PendingTarget)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func compareManifestWatermarks(files []SnapshotFile, watermarks []snapshotWatermark) error {
|
||||||
|
manifestByPath := make(map[string]SnapshotFile, len(files))
|
||||||
|
for _, file := range files {
|
||||||
|
key := snapshotPathKey(filepath.FromSlash(file.Path))
|
||||||
|
if _, exists := manifestByPath[key]; exists {
|
||||||
|
return fmt.Errorf("verify snapshot duplicate manifest path: %s", file.Path)
|
||||||
|
}
|
||||||
|
manifestByPath[key] = file
|
||||||
|
}
|
||||||
|
if len(manifestByPath) != len(watermarks) {
|
||||||
|
return fmt.Errorf("verify snapshot file/watermark count differs: manifest=%d database=%d", len(manifestByPath), len(watermarks))
|
||||||
|
}
|
||||||
|
for _, watermark := range watermarks {
|
||||||
|
path := filepath.Join("backup", watermark.RelativePath)
|
||||||
|
key := snapshotPathKey(path)
|
||||||
|
file, ok := manifestByPath[key]
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("verify snapshot manifest missing DB watermark: %s", filepath.ToSlash(path))
|
||||||
|
}
|
||||||
|
if file.Bytes != watermark.Bytes || file.IndexRecords != watermark.IndexRecords || file.AccountID != watermark.AccountID || file.Folder != watermark.Folder {
|
||||||
|
return fmt.Errorf("verify snapshot watermark mismatch: %s", file.Path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func verifySnapshotReindex(root, databasePath string, expectedIndexRecords int64, expectedIndexSHA256 string) error {
|
||||||
|
temporaryRoot, err := os.MkdirTemp("", "mail-graveyard-snapshot-verify-")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("verify snapshot temporary restore: %w", err)
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(temporaryRoot)
|
||||||
|
temporaryDB := filepath.Join(temporaryRoot, "mail-graveyard.db")
|
||||||
|
databaseInfo, err := os.Stat(databasePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("verify snapshot temporary database stat: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := copyFilePrefix(databasePath, temporaryDB, databaseInfo.Size()); err != nil {
|
||||||
|
return fmt.Errorf("verify snapshot temporary database copy: %w", err)
|
||||||
|
}
|
||||||
|
oldCfg, oldDB := Cfg, DB
|
||||||
|
Cfg.DBPath = temporaryDB
|
||||||
|
Cfg.MboxRoot = filepath.Join(root, "backup")
|
||||||
|
DB = nil
|
||||||
|
defer func() {
|
||||||
|
if DB != nil {
|
||||||
|
_ = DB.Close()
|
||||||
|
}
|
||||||
|
Cfg, DB = oldCfg, oldDB
|
||||||
|
}()
|
||||||
|
if err := ConnectDB(false); err != nil {
|
||||||
|
return fmt.Errorf("verify snapshot restore database: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := DB.Exec(`CREATE TABLE snapshot_verify_expected_index AS
|
||||||
|
SELECT account_id, folder, seq, message_id, body_sha256 FROM mbox_index`); err != nil {
|
||||||
|
return fmt.Errorf("verify snapshot preserve expected index: %w", err)
|
||||||
|
}
|
||||||
|
if err := ReindexArchives("all"); err != nil {
|
||||||
|
return fmt.Errorf("verify snapshot restore reindex: %w", err)
|
||||||
|
}
|
||||||
|
var reindexed int64
|
||||||
|
if err := DB.QueryRow(`SELECT count(*) FROM mbox_index`).Scan(&reindexed); err != nil {
|
||||||
|
return fmt.Errorf("verify snapshot restore index count: %w", err)
|
||||||
|
}
|
||||||
|
if reindexed != expectedIndexRecords {
|
||||||
|
return fmt.Errorf("verify snapshot restore index count=%d, want %d", reindexed, expectedIndexRecords)
|
||||||
|
}
|
||||||
|
reindexedDigest, err := indexDigest(DB)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if reindexedDigest != expectedIndexSHA256 {
|
||||||
|
difference, differenceErr := snapshotIndexDifference(DB)
|
||||||
|
if differenceErr != nil {
|
||||||
|
return fmt.Errorf("verify snapshot restore index digest=%s, want %s (difference query: %v)", reindexedDigest, expectedIndexSHA256, differenceErr)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("verify snapshot restore index digest=%s, want %s (%s)", reindexedDigest, expectedIndexSHA256, difference)
|
||||||
|
}
|
||||||
|
stats, err := snapshotDatabaseStats(DB)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if stats.MissingArchiveRecords != 0 {
|
||||||
|
return fmt.Errorf("verify snapshot restore has %d copied identities without archive index", stats.MissingArchiveRecords)
|
||||||
|
}
|
||||||
|
if stats.InvalidStageOrder != 0 {
|
||||||
|
return fmt.Errorf("verify snapshot restore has %d precise identities with invalid stage order", stats.InvalidStageOrder)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func snapshotIndexDifference(db *sql.DB) (string, error) {
|
||||||
|
type differenceRow struct {
|
||||||
|
accountID int64
|
||||||
|
folder string
|
||||||
|
seq int64
|
||||||
|
messageID string
|
||||||
|
bodyHash string
|
||||||
|
}
|
||||||
|
queries := []struct {
|
||||||
|
label string
|
||||||
|
left string
|
||||||
|
right string
|
||||||
|
}{
|
||||||
|
{"missing after reindex", "snapshot_verify_expected_index", "mbox_index"},
|
||||||
|
{"unexpected after reindex", "mbox_index", "snapshot_verify_expected_index"},
|
||||||
|
}
|
||||||
|
parts := make([]string, 0, len(queries))
|
||||||
|
for _, query := range queries {
|
||||||
|
statement := fmt.Sprintf(`SELECT account_id, folder, seq, message_id, body_sha256 FROM %s
|
||||||
|
EXCEPT SELECT account_id, folder, seq, message_id, body_sha256 FROM %s`, query.left, query.right)
|
||||||
|
var count int64
|
||||||
|
if err := db.QueryRow(`SELECT count(*) FROM (` + statement + `)`).Scan(&count); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
rows, err := db.Query(statement + ` LIMIT 3`)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
var samples []string
|
||||||
|
for rows.Next() {
|
||||||
|
var row differenceRow
|
||||||
|
if err := rows.Scan(&row.accountID, &row.folder, &row.seq, &row.messageID, &row.bodyHash); err != nil {
|
||||||
|
_ = rows.Close()
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
hash := row.bodyHash
|
||||||
|
if len(hash) > 16 {
|
||||||
|
hash = hash[:16]
|
||||||
|
}
|
||||||
|
samples = append(samples, fmt.Sprintf("account=%d folder=%q seq=%d id=%q hash=%s", row.accountID, row.folder, row.seq, row.messageID, hash))
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
_ = rows.Close()
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := rows.Close(); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
parts = append(parts, fmt.Sprintf("%s=%d samples=[%s]", query.label, count, strings.Join(samples, "; ")))
|
||||||
|
}
|
||||||
|
return strings.Join(parts, ", "), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func pathInsideSnapshot(root, relative string) (string, error) {
|
||||||
|
if relative == "" || filepath.IsAbs(relative) {
|
||||||
|
return "", fmt.Errorf("verify snapshot invalid relative path: %q", relative)
|
||||||
|
}
|
||||||
|
path := filepath.Join(root, filepath.FromSlash(relative))
|
||||||
|
pathAbs, err := filepath.Abs(path)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
rel, err := filepath.Rel(root, pathAbs)
|
||||||
|
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
|
||||||
|
return "", fmt.Errorf("verify snapshot path escapes root: %q", relative)
|
||||||
|
}
|
||||||
|
return pathAbs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func hashFile(path string) (string, error) {
|
||||||
|
f, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
hash := sha256.New()
|
||||||
|
if _, err := io.Copy(hash, f); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return hex.EncodeToString(hash.Sum(nil)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func syncFile(path string) error {
|
||||||
|
f, err := os.OpenFile(path, os.O_RDWR, 0)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
return f.Sync()
|
||||||
|
}
|
||||||
|
|
||||||
|
func snapshotBytes(files []SnapshotFile) int64 {
|
||||||
|
var total int64
|
||||||
|
for _, file := range files {
|
||||||
|
total += file.Bytes
|
||||||
|
}
|
||||||
|
return total
|
||||||
|
}
|
||||||
|
|
||||||
|
func snapshotPathKey(path string) string {
|
||||||
|
path = filepath.Clean(path)
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
return strings.ToLower(path)
|
||||||
|
}
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
|
||||||
|
func sortSnapshotFiles(files []SnapshotFile) {
|
||||||
|
sort.Slice(files, func(i, j int) bool { return files[i].Path < files[j].Path })
|
||||||
|
}
|
||||||
209
backend/14-snapshot_test.go
Normal file
209
backend/14-snapshot_test.go
Normal file
|
|
@ -0,0 +1,209 @@
|
||||||
|
package backend
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSnapshotUsesDatabaseWatermarkWhileArchiveKeepsGrowing(t *testing.T) {
|
||||||
|
for _, compression := range []string{"none", "zstd"} {
|
||||||
|
t.Run(compression, func(t *testing.T) {
|
||||||
|
account, writer, restore := setupSnapshotTest(t, compression)
|
||||||
|
defer restore()
|
||||||
|
|
||||||
|
first := RawMessage{
|
||||||
|
MessageID: "first@example.com",
|
||||||
|
Body: []byte("Message-ID: <first@example.com>\r\nSubject: First\r\n\r\nFirst body\r\nFrom quoted\r\n"),
|
||||||
|
InternalDate: time.Unix(1_700_000_000, 0),
|
||||||
|
}
|
||||||
|
firstInfo := appendSnapshotTestMessage(t, account, writer, "INBOX", first)
|
||||||
|
snapshotRoot := filepath.Join(t.TempDir(), "snapshot")
|
||||||
|
var secondInfo MboxAppendInfo
|
||||||
|
err := createSnapshot(snapshotRoot, func() error {
|
||||||
|
second := RawMessage{
|
||||||
|
MessageID: "second@example.com",
|
||||||
|
Body: []byte("Message-ID: <second@example.com>\r\nSubject: Second\r\n\r\nSecond body"),
|
||||||
|
InternalDate: time.Unix(1_700_000_100, 0),
|
||||||
|
}
|
||||||
|
secondInfo = appendSnapshotTestMessage(t, account, writer, "INBOX", second)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
manifest := readSnapshotTestManifest(t, snapshotRoot)
|
||||||
|
if runtime.GOOS != "windows" {
|
||||||
|
databaseInfo, err := os.Stat(filepath.Join(snapshotRoot, filepath.FromSlash(manifest.DatabasePath)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if permissions := databaseInfo.Mode().Perm(); permissions != 0o600 {
|
||||||
|
t.Fatalf("snapshot database permissions=%#o, want 0600", permissions)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if manifest.IndexRecords != 1 || manifest.PreciseCopied != 1 || len(manifest.Files) != 1 {
|
||||||
|
t.Fatalf("snapshot crossed DB boundary: %#v", manifest)
|
||||||
|
}
|
||||||
|
if manifest.Files[0].Bytes != firstInfo.FileOffset+firstInfo.FrameLen {
|
||||||
|
t.Fatalf("snapshot bytes=%d, want first committed boundary=%d", manifest.Files[0].Bytes, firstInfo.FileOffset+firstInfo.FrameLen)
|
||||||
|
}
|
||||||
|
sourceInfo, err := os.Stat(secondInfo.Path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if sourceInfo.Size() <= manifest.Files[0].Bytes {
|
||||||
|
t.Fatalf("source did not grow beyond snapshot boundary: source=%d snapshot=%d", sourceInfo.Size(), manifest.Files[0].Bytes)
|
||||||
|
}
|
||||||
|
snapshotMbox := filepath.Join(snapshotRoot, filepath.FromSlash(manifest.Files[0].Path))
|
||||||
|
messages, err := readMboxMessages(snapshotMbox)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(messages) != 1 || !strings.Contains(string(messages[0]), "First body") {
|
||||||
|
t.Fatalf("snapshot messages=%d, expected only first message", len(messages))
|
||||||
|
}
|
||||||
|
if err := VerifySnapshot(snapshotRoot); err != nil {
|
||||||
|
t.Fatalf("verify valid snapshot: %v", err)
|
||||||
|
}
|
||||||
|
var liveIndex int
|
||||||
|
if err := DB.QueryRow(`SELECT count(*) FROM mbox_index WHERE account_id=?`, account.ID).Scan(&liveIndex); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if liveIndex != 2 {
|
||||||
|
t.Fatalf("snapshot verification altered live database: index=%d", liveIndex)
|
||||||
|
}
|
||||||
|
|
||||||
|
f, err := os.OpenFile(snapshotMbox, os.O_APPEND|os.O_WRONLY, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := f.Write([]byte("tamper")); err != nil {
|
||||||
|
_ = f.Close()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := f.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := VerifySnapshot(snapshotRoot); err == nil {
|
||||||
|
t.Fatal("verification accepted a modified mbox")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSnapshotRefusesKnownMissingArchiveIdentity(t *testing.T) {
|
||||||
|
account, _, restore := setupSnapshotTest(t, "none")
|
||||||
|
defer restore()
|
||||||
|
identity := MessageIdentity{MessageID: "missing@example.com", BodySHA256: bodySHA256([]byte("missing"))}
|
||||||
|
if err := MarkIdentityCopied(account.ID, "INBOX", identity); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
target := filepath.Join(t.TempDir(), "refused")
|
||||||
|
err := CreateSnapshot(target)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "no archive index") {
|
||||||
|
t.Fatalf("snapshot error=%v, want missing archive refusal", err)
|
||||||
|
}
|
||||||
|
if _, statErr := os.Stat(target); !os.IsNotExist(statErr) {
|
||||||
|
t.Fatalf("refused snapshot unexpectedly finalized: %v", statErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSnapshotDoesNotOverwriteExistingTarget(t *testing.T) {
|
||||||
|
_, _, restore := setupSnapshotTest(t, "none")
|
||||||
|
defer restore()
|
||||||
|
target := filepath.Join(t.TempDir(), "existing")
|
||||||
|
if err := os.Mkdir(target, 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sentinel := filepath.Join(target, "keep")
|
||||||
|
if err := os.WriteFile(sentinel, []byte("keep"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := CreateSnapshot(target); err == nil {
|
||||||
|
t.Fatal("snapshot overwrote an existing target")
|
||||||
|
}
|
||||||
|
if got, err := os.ReadFile(sentinel); err != nil || string(got) != "keep" {
|
||||||
|
t.Fatalf("existing target changed: %q %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func setupSnapshotTest(t *testing.T, compression string) (Account, *MboxWriter, func()) {
|
||||||
|
t.Helper()
|
||||||
|
oldCfg, oldDB := Cfg, DB
|
||||||
|
root := t.TempDir()
|
||||||
|
Cfg = Config{
|
||||||
|
DBPath: filepath.Join(root, "mail-graveyard.db"),
|
||||||
|
MboxRoot: filepath.Join(root, "backup"),
|
||||||
|
MboxCompression: compression,
|
||||||
|
}
|
||||||
|
DB = nil
|
||||||
|
if err := ConnectDB(true); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
restore := func() {
|
||||||
|
if DB != nil {
|
||||||
|
_ = DB.Close()
|
||||||
|
}
|
||||||
|
Cfg, DB = oldCfg, oldDB
|
||||||
|
}
|
||||||
|
if err := SaveAccount(Account{
|
||||||
|
Name: "snapshot-account", SrcHost: "source.example", SrcPort: 993, SrcSecurity: "tls", SrcUser: "source@example.com", SrcPass: "x",
|
||||||
|
DstHost: "target.example", DstPort: 993, DstSecurity: "tls", DstUser: "target@example.com", DstPass: "x",
|
||||||
|
MboxDir: "snapshot@example.com", Active: true,
|
||||||
|
}); err != nil {
|
||||||
|
restore()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
account, err := GetAccount("snapshot-account")
|
||||||
|
if err != nil {
|
||||||
|
restore()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
writer, err := NewMboxWriter(filepath.Join(Cfg.MboxRoot, account.MboxDir))
|
||||||
|
if err != nil {
|
||||||
|
restore()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return account, writer, restore
|
||||||
|
}
|
||||||
|
|
||||||
|
func appendSnapshotTestMessage(t *testing.T, account Account, writer *MboxWriter, folder string, message RawMessage) MboxAppendInfo {
|
||||||
|
t.Helper()
|
||||||
|
identity := identityForRawMessage(message)
|
||||||
|
info, err := writer.Append(folder, message)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := SaveMboxIndex(MboxIndexEntry{
|
||||||
|
AccountID: account.ID, Folder: safeMboxName(folder), MessageID: identity.MessageID, BodySHA256: identity.BodySHA256,
|
||||||
|
FileOffset: info.FileOffset, FrameLen: info.FrameLen, InnerOffset: info.InnerOffset, InnerLen: info.InnerLen,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := UpdateMboxIndexState(account.ID, safeMboxName(folder), info.FileOffset+info.FrameLen); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := MarkIdentityCopied(account.ID, safeMboxName(folder), identity); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return info
|
||||||
|
}
|
||||||
|
|
||||||
|
func readSnapshotTestManifest(t *testing.T, root string) SnapshotManifest {
|
||||||
|
t.Helper()
|
||||||
|
b, err := os.ReadFile(filepath.Join(root, "snapshot-manifest.json"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var manifest SnapshotManifest
|
||||||
|
if err := json.Unmarshal(b, &manifest); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return manifest
|
||||||
|
}
|
||||||
16
main.go
16
main.go
|
|
@ -25,16 +25,32 @@ func main() {
|
||||||
checkAccount := flag.String("check-account", "", "Konto-Logins pruefen und INBOX zaehlen, ohne zu kopieren")
|
checkAccount := flag.String("check-account", "", "Konto-Logins pruefen und INBOX zaehlen, ohne zu kopieren")
|
||||||
reindex := flag.String("reindex", "", "Archiv-mbox-Index fuer Konto/Archiv oder 'all' neu aufbauen")
|
reindex := flag.String("reindex", "", "Archiv-mbox-Index fuer Konto/Archiv oder 'all' neu aufbauen")
|
||||||
dedupTarget := flag.String("dedup-target", "", "Ziel-Postfach nach Message-ID-Dubletten scannen (Dry-Run ohne --apply)")
|
dedupTarget := flag.String("dedup-target", "", "Ziel-Postfach nach Message-ID-Dubletten scannen (Dry-Run ohne --apply)")
|
||||||
|
snapshot := flag.String("snapshot", "", "konsistenten DB- und mbox-Praefix-Snapshot in ein neues Zielverzeichnis schreiben")
|
||||||
|
verifySnapshot := flag.String("verify-snapshot", "", "Snapshot isoliert pruefen und dessen DB-Kopie vollstaendig reindexieren")
|
||||||
apply := flag.Bool("apply", false, "Scharfe Ausfuehrung fuer riskante Wartungsbefehle wie --dedup-target")
|
apply := flag.Bool("apply", false, "Scharfe Ausfuehrung fuer riskante Wartungsbefehle wie --dedup-target")
|
||||||
initDB := flag.Bool("init-db", false, "SQLite-DB neu anlegen, falls db_path noch nicht existiert")
|
initDB := flag.Bool("init-db", false, "SQLite-DB neu anlegen, falls db_path noch nicht existiert")
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
if *snapshot != "" && (*verifySnapshot != "" || *runOnce != "" || *seedAccount != "" || *checkAccount != "" || *reindex != "" || *dedupTarget != "" || *watch) {
|
||||||
|
log.Fatal("-snapshot kann nicht mit einem anderen Betriebsmodus kombiniert werden")
|
||||||
|
}
|
||||||
|
if *verifySnapshot != "" && (*runOnce != "" || *seedAccount != "" || *checkAccount != "" || *reindex != "" || *dedupTarget != "" || *watch) {
|
||||||
|
log.Fatal("-verify-snapshot kann nicht mit einem anderen Betriebsmodus kombiniert werden")
|
||||||
|
}
|
||||||
|
|
||||||
if err := backend.LoadConfig("config.json"); err != nil {
|
if err := backend.LoadConfig("config.json"); err != nil {
|
||||||
log.Fatalf("config: %v", err)
|
log.Fatalf("config: %v", err)
|
||||||
}
|
}
|
||||||
|
if *verifySnapshot != "" {
|
||||||
|
must(backend.VerifySnapshot(*verifySnapshot))
|
||||||
|
return
|
||||||
|
}
|
||||||
if err := backend.ConnectDB(*initDB); err != nil {
|
if err := backend.ConnectDB(*initDB); err != nil {
|
||||||
log.Fatalf("database: %v", err)
|
log.Fatalf("database: %v", err)
|
||||||
}
|
}
|
||||||
|
if *snapshot != "" {
|
||||||
|
must(backend.CreateSnapshot(*snapshot))
|
||||||
|
return
|
||||||
|
}
|
||||||
if err := backend.InitAuth(); err != nil {
|
if err := backend.InitAuth(); err != nil {
|
||||||
log.Fatalf("auth: %v", err)
|
log.Fatalf("auth: %v", err)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue