package backend import ( "bytes" "database/sql" "os" "path/filepath" "strings" "testing" "time" "github.com/klauspost/compress/zstd" ) func TestZstdMboxRoundTripAndIndexRead(t *testing.T) { oldCfg, oldDB := Cfg, DB root := t.TempDir() t.Cleanup(func() { if DB != nil { _ = DB.Close() } Cfg, DB = oldCfg, oldDB }) Cfg = Config{ DBPath: filepath.Join(root, "mail-graveyard.db"), MboxRoot: filepath.Join(root, "backup"), MboxCompression: "zstd", } if err := ConnectDB(true); err != nil { t.Fatal(err) } if err := SaveArchiveMailbox("archive"); err != nil { t.Fatal(err) } if err := SaveAccount(Account{ Name: "test-account", SrcHost: "source.example", SrcPort: 993, SrcSecurity: "tls", SrcUser: "source@example.com", SrcPass: "x", DstHost: "target.example", DstPort: 993, DstSecurity: "tls", DstUser: "target@example.com", DstPass: "x", MboxDir: "archive", Active: true, }); err != nil { t.Fatal(err) } account, err := GetAccount("test-account") if err != nil { t.Fatal(err) } msg := RawMessage{ MessageID: "zstd-test@example.com", InternalDate: time.Date(2026, 7, 14, 10, 11, 12, 0, time.UTC), Body: []byte("Message-ID: \r\n" + "From: Sender \r\n" + "Subject: Zstd Test\r\n" + "Date: Tue, 14 Jul 2026 10:11:12 +0000\r\n" + "\r\n" + "Hello\r\nFrom inside body\r\n"), } writer, err := NewMboxWriter(filepath.Join(Cfg.MboxRoot, "archive")) if err != nil { t.Fatal(err) } info, err := writer.Append("INBOX", msg) if err != nil { t.Fatal(err) } if err := SaveMboxIndex(MboxIndexEntry{ AccountID: account.ID, Folder: "INBOX", MessageID: msg.MessageID, Subject: info.Subject, From: info.From, Date: info.Date, FileOffset: info.FileOffset, FrameLen: info.FrameLen, InnerOffset: info.InnerOffset, InnerLen: info.InnerLen, }); err != nil { t.Fatal(err) } compressed, err := os.ReadFile(info.Path) if err != nil { t.Fatal(err) } dec, err := zstd.NewReader(nil) if err != nil { t.Fatal(err) } defer dec.Close() plain, err := dec.DecodeAll(compressed, nil) if err != nil { t.Fatal(err) } if want := mboxRecord(msg); !bytes.Equal(plain, want) { t.Fatalf("zstd round-trip differs: got %d bytes, want %d", len(plain), len(want)) } entries, err := ReadMboxList(info.Path) if err != nil { t.Fatal(err) } if len(entries) != 1 || entries[0].Index != 0 || entries[0].Subject != "Zstd Test" { t.Fatalf("unexpected index list: %#v", entries) } raw, err := ReadMboxMessage(info.Path, entries[0].Index) if err != nil { t.Fatal(err) } if !bytes.Contains(raw, []byte("Hello")) || bytes.Contains(raw, []byte(">From inside body")) { t.Fatalf("unexpected message body: %q", raw) } } func TestPlainMboxPartialIndexIsRebuiltBeforeList(t *testing.T) { oldCfg, oldDB := Cfg, DB root := t.TempDir() t.Cleanup(func() { if DB != nil { _ = DB.Close() } Cfg, DB = oldCfg, oldDB }) Cfg = Config{ DBPath: filepath.Join(root, "mail-graveyard.db"), MboxRoot: filepath.Join(root, "backup"), MboxCompression: "none", } if err := ConnectDB(true); err != nil { t.Fatal(err) } if err := SaveArchiveMailbox("archive"); err != nil { t.Fatal(err) } if err := SaveAccount(Account{ Name: "test-account", SrcHost: "source.example", SrcPort: 993, SrcSecurity: "tls", SrcUser: "source@example.com", SrcPass: "x", DstHost: "target.example", DstPort: 993, DstSecurity: "tls", DstUser: "target@example.com", DstPass: "x", MboxDir: "archive", Active: true, }); err != nil { t.Fatal(err) } account, err := GetAccount("test-account") if err != nil { t.Fatal(err) } writer, err := NewMboxWriter(filepath.Join(Cfg.MboxRoot, "archive")) if err != nil { t.Fatal(err) } first := testRawMessage("first@example.com", "First") firstInfo, err := writer.Append("INBOX", first) if err != nil { t.Fatal(err) } if err := SaveMboxIndex(MboxIndexEntry{ AccountID: account.ID, Folder: "INBOX", MessageID: first.MessageID, Subject: firstInfo.Subject, From: firstInfo.From, Date: firstInfo.Date, FileOffset: firstInfo.FileOffset, FrameLen: firstInfo.FrameLen, InnerOffset: firstInfo.InnerOffset, InnerLen: firstInfo.InnerLen, }); err != nil { t.Fatal(err) } if err := UpdateMboxIndexState(account.ID, "INBOX", firstInfo.FileOffset+firstInfo.FrameLen); err != nil { t.Fatal(err) } if _, err := writer.Append("INBOX", testRawMessage("second@example.com", "Second")); err != nil { t.Fatal(err) } entries, err := ReadMboxList(firstInfo.Path) if err != nil { t.Fatal(err) } if len(entries) != 2 { t.Fatalf("expected rebuilt full list with 2 entries, got %#v", entries) } if entries[0].Subject != "Second" || entries[1].Subject != "First" { t.Fatalf("unexpected entries after reindex: %#v", entries) } } func TestReadMboxMessagesBytesClonesBufferRecords(t *testing.T) { mbox := []byte(strings.Join([]string{ "From one@example.com Tue Jul 14 10:00:00 2026", "Message-ID: ", "Subject: One", "", "short", "From two@example.com Tue Jul 14 10:01:00 2026", "Message-ID: ", "Subject: Two", "", "this message is deliberately much longer than the first one", "and has another line", "From three@example.com Tue Jul 14 10:02:00 2026", "Message-ID: ", "Subject: Three", "", "tiny", "", }, "\n")) msgs := readMboxMessagesBytes(mbox) if len(msgs) != 3 { t.Fatalf("expected 3 messages, got %d", len(msgs)) } wants := []string{"Subject: One", "Subject: Two", "Subject: Three"} for i, want := range wants { if !bytes.Contains(msgs[i], []byte(want)) { t.Fatalf("message %d does not contain %q: %q", i, want, msgs[i]) } } if bytes.Contains(msgs[0], []byte("Subject: Two")) || bytes.Contains(msgs[1], []byte("Subject: Three")) { t.Fatalf("messages share buffer contents: %#q", msgs) } } func TestPlainIndexedReadUsesOffsetAcrossMissingMiddleIndex(t *testing.T) { oldCfg, oldDB := Cfg, DB root := t.TempDir() t.Cleanup(func() { if DB != nil { _ = DB.Close() } Cfg, DB = oldCfg, oldDB }) Cfg = Config{DBPath: filepath.Join(root, "mail-graveyard.db"), MboxRoot: filepath.Join(root, "backup")} if err := ConnectDB(true); err != nil { t.Fatal(err) } if err := SaveArchiveMailbox("archive"); err != nil { t.Fatal(err) } if err := SaveAccount(Account{ Name: "test-account", SrcHost: "source.example", SrcPort: 993, SrcSecurity: "tls", SrcUser: "source@example.com", SrcPass: "x", DstHost: "target.example", DstPort: 993, DstSecurity: "tls", DstUser: "target@example.com", DstPass: "x", MboxDir: "archive", Active: true, }); err != nil { t.Fatal(err) } account, err := GetAccount("test-account") if err != nil { t.Fatal(err) } writer, err := NewMboxWriter(filepath.Join(Cfg.MboxRoot, "archive")) if err != nil { t.Fatal(err) } firstInfo, err := writer.Append("INBOX", testRawMessage("first@example.com", "First")) if err != nil { t.Fatal(err) } if err := SaveMboxIndex(MboxIndexEntry{AccountID: account.ID, Folder: "INBOX", MessageID: "first@example.com", Subject: "First", FileOffset: firstInfo.FileOffset, FrameLen: firstInfo.FrameLen, InnerOffset: firstInfo.InnerOffset, InnerLen: firstInfo.InnerLen}); err != nil { t.Fatal(err) } if _, err := writer.Append("INBOX", testRawMessage("missing@example.com", "Missing middle")); err != nil { t.Fatal(err) } thirdInfo, err := writer.Append("INBOX", testRawMessage("third@example.com", "Third")) if err != nil { t.Fatal(err) } if err := SaveMboxIndex(MboxIndexEntry{AccountID: account.ID, Folder: "INBOX", MessageID: "third@example.com", Subject: "Third", FileOffset: thirdInfo.FileOffset, FrameLen: thirdInfo.FrameLen, InnerOffset: thirdInfo.InnerOffset, InnerLen: thirdInfo.InnerLen}); err != nil { t.Fatal(err) } if err := UpdateMboxIndexState(account.ID, "INBOX", thirdInfo.FileOffset+thirdInfo.FrameLen); err != nil { t.Fatal(err) } raw, err := ReadMboxMessage(thirdInfo.Path, 1) if err != nil { t.Fatal(err) } if !bytes.Contains(raw, []byte("Subject: Third")) || bytes.Contains(raw, []byte("Missing middle")) { t.Fatalf("indexed read drifted to the file position: %q", raw) } if err := ReindexMboxFile(account.ID, "INBOX", thirdInfo.Path); err != nil { t.Fatal(err) } for i, subject := range []string{"First", "Missing middle", "Third"} { raw, err := ReadMboxMessage(thirdInfo.Path, i) if err != nil { t.Fatalf("rebuilt index %d: %v", i, err) } if !bytes.Contains(raw, []byte("Subject: "+subject)) { t.Fatalf("rebuilt index %d returned wrong message: %q", i, raw) } } } func TestConnectDBConfiguresBusyTimeoutAndCopyStages(t *testing.T) { oldCfg, oldDB := Cfg, DB root := t.TempDir() t.Cleanup(func() { if DB != nil { _ = DB.Close() } Cfg, DB = oldCfg, oldDB }) Cfg = Config{DBPath: filepath.Join(root, "mail-graveyard.db"), MboxRoot: filepath.Join(root, "backup")} if err := ConnectDB(true); err != nil { t.Fatal(err) } var timeout int if err := DB.QueryRow(`PRAGMA busy_timeout`).Scan(&timeout); err != nil { t.Fatal(err) } if timeout != 10000 { t.Fatalf("busy_timeout=%d, want 10000", timeout) } if err := SaveAccount(Account{ Name: "stage-account", SrcHost: "source.example", SrcPort: 993, SrcSecurity: "tls", SrcUser: "source@example.com", SrcPass: "x", DstHost: "target.example", DstPort: 993, DstSecurity: "tls", DstUser: "target@example.com", DstPass: "x", Active: true, }); err != nil { t.Fatal(err) } account, err := GetAccount("stage-account") if err != nil { t.Fatal(err) } if err := MarkMboxCopied(account.ID, "INBOX", "stage@example.com"); err != nil { t.Fatal(err) } state, err := GetCopyState(account.ID, "INBOX", "stage@example.com") if err != nil { t.Fatal(err) } if !state.MboxDone || state.TargetDone { t.Fatalf("unexpected mbox-only state: %#v", state) } if err := MarkTargetCopied(account.ID, "INBOX", "stage@example.com"); err != nil { t.Fatal(err) } state, err = GetCopyState(account.ID, "INBOX", "stage@example.com") if err != nil { t.Fatal(err) } if !state.MboxDone || !state.TargetDone { t.Fatalf("unexpected completed state: %#v", state) } first := MessageIdentity{MessageID: "reused@example.com", BodySHA256: bodySHA256([]byte("first"))} second := MessageIdentity{MessageID: "reused@example.com", BodySHA256: bodySHA256([]byte("second"))} if err := MarkIdentityCopied(account.ID, "INBOX", first); err != nil { t.Fatal(err) } state, err = GetCopyIdentityState(account.ID, "INBOX", second) if err != nil { t.Fatal(err) } if state.MboxDone || state.TargetDone { t.Fatalf("byte-different message with reused ID was hidden: %#v", state) } } func TestIdentitySchemaUpgradePreservesLegacyAliases(t *testing.T) { oldCfg, oldDB := Cfg, DB root := t.TempDir() path := filepath.Join(root, "legacy.db") rawDB, err := sql.Open("sqlite", path) if err != nil { t.Fatal(err) } legacySchema := []string{ `CREATE TABLE accounts(id INTEGER PRIMARY KEY)`, `INSERT INTO accounts(id) VALUES(7)`, `CREATE TABLE copied(account_id INTEGER NOT NULL, folder TEXT NOT NULL, message_id TEXT NOT NULL, mbox_done INTEGER NOT NULL DEFAULT 1, target_done INTEGER NOT NULL DEFAULT 1, copied_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, UNIQUE(account_id, folder, message_id))`, `INSERT INTO copied(account_id, folder, message_id, mbox_done, target_done) VALUES(7, 'INBOX.Newbies ', 'sha256:legacy-raw', 1, 0)`, `CREATE TABLE mbox_index(account_id INTEGER NOT NULL, folder TEXT NOT NULL, seq INTEGER NOT NULL, message_id TEXT NOT NULL, subject TEXT NOT NULL DEFAULT '', from_addr TEXT NOT NULL DEFAULT '', date TEXT NOT NULL DEFAULT '', file_offset INTEGER NOT NULL, frame_len INTEGER NOT NULL, inner_offset INTEGER NOT NULL DEFAULT 0, inner_len INTEGER NOT NULL, UNIQUE(account_id, folder, message_id))`, `INSERT INTO mbox_index(account_id, folder, seq, message_id, file_offset, frame_len, inner_len) VALUES(7, 'INBOX.Newbies ', 0, 'sha256:legacy-raw', 0, 10, 10)`, `CREATE TABLE mbox_index_state(account_id INTEGER NOT NULL, folder TEXT NOT NULL, indexed_bytes INTEGER NOT NULL DEFAULT 0, UNIQUE(account_id, folder))`, `INSERT INTO mbox_index_state(account_id, folder, indexed_bytes) VALUES(7, 'INBOX.Newbies ', 10)`, } for _, stmt := range legacySchema { if _, err := rawDB.Exec(stmt); err != nil { _ = rawDB.Close() t.Fatal(err) } } if err := rawDB.Close(); err != nil { t.Fatal(err) } if DB != nil { _ = DB.Close() } t.Cleanup(func() { if DB != nil { _ = DB.Close() } Cfg, DB = oldCfg, oldDB }) Cfg = Config{DBPath: path, MboxRoot: filepath.Join(root, "backup")} if err := ConnectDB(false); err != nil { t.Fatal(err) } var messageID, bodyHash string var mboxDone, targetDone int if err := DB.QueryRow(`SELECT message_id, body_sha256, mbox_done, target_done FROM copied WHERE account_id=7`).Scan(&messageID, &bodyHash, &mboxDone, &targetDone); err != nil { t.Fatal(err) } if messageID != "sha256:legacy-raw" || bodyHash != "" || mboxDone != 1 || targetDone != 0 { t.Fatalf("legacy alias changed during upgrade: id=%q hash=%q stages=%d/%d", messageID, bodyHash, mboxDone, targetDone) } var indexedBytes int64 if err := DB.QueryRow(`SELECT indexed_bytes FROM mbox_index_state WHERE account_id=7 AND folder='INBOX.Newbies '`).Scan(&indexedBytes); err != nil { t.Fatal(err) } if indexedBytes != -1 { t.Fatalf("identity-less index was not invalidated: %d", indexedBytes) } stableHash := bodySHA256([]byte("archived body")) if err := ReplaceMboxIndex(7, "INBOX.Newbies", []MboxIndexEntry{{BodySHA256: stableHash, FrameLen: 10, InnerLen: 10}}, 10); err != nil { t.Fatal(err) } if err := DB.QueryRow(`SELECT mbox_done, target_done FROM copied WHERE account_id=7 AND folder='INBOX.Newbies' AND message_id='' AND body_sha256=?`, stableHash).Scan(&mboxDone, &targetDone); err != nil { t.Fatal(err) } if mboxDone != 1 || targetDone != 0 { t.Fatalf("reindex lost pending target state from legacy alias: %d/%d", mboxDone, targetDone) } if err := RemoveMboxIndexFolderAliases(7, map[string]bool{"INBOX.Newbies": true}); err != nil { t.Fatal(err) } var staleIndex, preservedAlias int if err := DB.QueryRow(`SELECT count(*) FROM mbox_index WHERE account_id=7 AND folder='INBOX.Newbies '`).Scan(&staleIndex); err != nil { t.Fatal(err) } if err := DB.QueryRow(`SELECT count(*) FROM copied WHERE account_id=7 AND folder='INBOX.Newbies ' AND message_id='sha256:legacy-raw'`).Scan(&preservedAlias); err != nil { t.Fatal(err) } if staleIndex != 0 || preservedAlias != 1 { t.Fatalf("folder alias cleanup stale_index=%d preserved_copy_alias=%d", staleIndex, preservedAlias) } } func TestHTMLToTextHandlesUnicodeBeforeCaseInsensitiveTags(t *testing.T) { tests := []struct { name string in string want []string drop []string }{ { name: "turkish dotted i", in: "

Turkish İ test


x", want: []string{"Turkish İ test", "x"}, }, { name: "kelvin sign", in: "
Kelvin K test

x", want: []string{"Kelvin K test", "x"}, }, { name: "mixed blocks and repeated replacements", in: "İ

first

second

third", want: []string{"İ", "first", "K", "second", "third"}, drop: []string{"bad"}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := htmlToText(tt.in) for _, want := range tt.want { if !strings.Contains(got, want) { t.Fatalf("htmlToText(%q) = %q, missing %q", tt.in, got, want) } } for _, drop := range tt.drop { if strings.Contains(got, drop) { t.Fatalf("htmlToText(%q) = %q, unexpectedly contains %q", tt.in, got, drop) } } }) } } func TestASCIIFoldIndexReturnsOriginalByteOffset(t *testing.T) { for _, tt := range []struct { s string needle string want int }{ {"İİ
x", "
", len("İİ")}, {"KK", "", len("KK")}, {"prefix
suffix", "
", len("prefix")}, } { if got := asciiFoldIndex(tt.s, tt.needle); got != tt.want { t.Fatalf("asciiFoldIndex(%q, %q) = %d, want %d", tt.s, tt.needle, got, tt.want) } } } func TestMessageBodyPanicFallsBackToRawMessage(t *testing.T) { raw := []byte("Subject: Evidence\r\n\r\nraw body") got := renderMessageBodySafely(raw, "test account=archive folder=INBOX seq=7", func() string { panic("synthetic parser failure") }) if !strings.Contains(got, "Darstellung fehlgeschlagen") || !strings.Contains(got, string(raw)) { t.Fatalf("panic fallback did not preserve raw message: %q", got) } } func testRawMessage(id, subject string) RawMessage { return RawMessage{ MessageID: id, InternalDate: time.Date(2026, 7, 14, 10, 11, 12, 0, time.UTC), Body: []byte("Message-ID: <" + id + ">\r\n" + "From: Sender \r\n" + "Subject: " + subject + "\r\n" + "Date: Tue, 14 Jul 2026 10:11:12 +0000\r\n" + "\r\n" + "Hello\r\n"), } }