Mail-Graveyard/backend/06-mbox_test.go

package backend

import (
	"bytes"
	"database/sql"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/klauspost/compress/zstd"
)

func TestZstdMboxRoundTripAndIndexRead(t *testing.T) {
	oldCfg, oldDB := Cfg, DB
	root := t.TempDir()
	t.Cleanup(func() {
		if DB != nil {
			_ = DB.Close()
		}
		Cfg, DB = oldCfg, oldDB
	})

	Cfg = Config{
		DBPath:          filepath.Join(root, "mail-graveyard.db"),
		MboxRoot:        filepath.Join(root, "backup"),
		MboxCompression: "zstd",
	}
	if err := ConnectDB(true); err != nil {
		t.Fatal(err)
	}
	if err := SaveArchiveMailbox("archive"); err != nil {
		t.Fatal(err)
	}
	if err := SaveAccount(Account{
		Name:        "test-account",
		SrcHost:     "source.example",
		SrcPort:     993,
		SrcSecurity: "tls",
		SrcUser:     "source@example.com",
		SrcPass:     "x",
		DstHost:     "target.example",
		DstPort:     993,
		DstSecurity: "tls",
		DstUser:     "target@example.com",
		DstPass:     "x",
		MboxDir:     "archive",
		Active:      true,
	}); err != nil {
		t.Fatal(err)
	}
	account, err := GetAccount("test-account")
	if err != nil {
		t.Fatal(err)
	}

	msg := RawMessage{
		MessageID:    "zstd-test@example.com",
		InternalDate: time.Date(2026, 7, 14, 10, 11, 12, 0, time.UTC),
		Body: []byte("Message-ID: <zstd-test@example.com>\r\n" +
			"From: Sender <sender@example.com>\r\n" +
			"Subject: Zstd Test\r\n" +
			"Date: Tue, 14 Jul 2026 10:11:12 +0000\r\n" +
			"\r\n" +
			"Hello\r\nFrom inside body\r\n"),
	}
	writer, err := NewMboxWriter(filepath.Join(Cfg.MboxRoot, "archive"))
	if err != nil {
		t.Fatal(err)
	}
	info, err := writer.Append("INBOX", msg)
	if err != nil {
		t.Fatal(err)
	}
	if err := SaveMboxIndex(MboxIndexEntry{
		AccountID:   account.ID,
		Folder:      "INBOX",
		MessageID:   msg.MessageID,
		Subject:     info.Subject,
		From:        info.From,
		Date:        info.Date,
		FileOffset:  info.FileOffset,
		FrameLen:    info.FrameLen,
		InnerOffset: info.InnerOffset,
		InnerLen:    info.InnerLen,
	}); err != nil {
		t.Fatal(err)
	}

	compressed, err := os.ReadFile(info.Path)
	if err != nil {
		t.Fatal(err)
	}
	dec, err := zstd.NewReader(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer dec.Close()
	plain, err := dec.DecodeAll(compressed, nil)
	if err != nil {
		t.Fatal(err)
	}
	if want := mboxRecord(msg); !bytes.Equal(plain, want) {
		t.Fatalf("zstd round-trip differs: got %d bytes, want %d", len(plain), len(want))
	}

	entries, err := ReadMboxList(info.Path)
	if err != nil {
		t.Fatal(err)
	}
	if len(entries) != 1 || entries[0].Index != 0 || entries[0].Subject != "Zstd Test" {
		t.Fatalf("unexpected index list: %#v", entries)
	}
	raw, err := ReadMboxMessage(info.Path, entries[0].Index)
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Contains(raw, []byte("Hello")) || bytes.Contains(raw, []byte(">From inside body")) {
		t.Fatalf("unexpected message body: %q", raw)
	}
}

func TestPlainMboxPartialIndexIsRebuiltBeforeList(t *testing.T) {
	oldCfg, oldDB := Cfg, DB
	root := t.TempDir()
	t.Cleanup(func() {
		if DB != nil {
			_ = DB.Close()
		}
		Cfg, DB = oldCfg, oldDB
	})

	Cfg = Config{
		DBPath:          filepath.Join(root, "mail-graveyard.db"),
		MboxRoot:        filepath.Join(root, "backup"),
		MboxCompression: "none",
	}
	if err := ConnectDB(true); err != nil {
		t.Fatal(err)
	}
	if err := SaveArchiveMailbox("archive"); err != nil {
		t.Fatal(err)
	}
	if err := SaveAccount(Account{
		Name:        "test-account",
		SrcHost:     "source.example",
		SrcPort:     993,
		SrcSecurity: "tls",
		SrcUser:     "source@example.com",
		SrcPass:     "x",
		DstHost:     "target.example",
		DstPort:     993,
		DstSecurity: "tls",
		DstUser:     "target@example.com",
		DstPass:     "x",
		MboxDir:     "archive",
		Active:      true,
	}); err != nil {
		t.Fatal(err)
	}
	account, err := GetAccount("test-account")
	if err != nil {
		t.Fatal(err)
	}
	writer, err := NewMboxWriter(filepath.Join(Cfg.MboxRoot, "archive"))
	if err != nil {
		t.Fatal(err)
	}
	first := testRawMessage("first@example.com", "First")
	firstInfo, err := writer.Append("INBOX", first)
	if err != nil {
		t.Fatal(err)
	}
	if err := SaveMboxIndex(MboxIndexEntry{
		AccountID:   account.ID,
		Folder:      "INBOX",
		MessageID:   first.MessageID,
		Subject:     firstInfo.Subject,
		From:        firstInfo.From,
		Date:        firstInfo.Date,
		FileOffset:  firstInfo.FileOffset,
		FrameLen:    firstInfo.FrameLen,
		InnerOffset: firstInfo.InnerOffset,
		InnerLen:    firstInfo.InnerLen,
	}); err != nil {
		t.Fatal(err)
	}
	if err := UpdateMboxIndexState(account.ID, "INBOX", firstInfo.FileOffset+firstInfo.FrameLen); err != nil {
		t.Fatal(err)
	}
	if _, err := writer.Append("INBOX", testRawMessage("second@example.com", "Second")); err != nil {
		t.Fatal(err)
	}

	entries, err := ReadMboxList(firstInfo.Path)
	if err != nil {
		t.Fatal(err)
	}
	if len(entries) != 2 {
		t.Fatalf("expected rebuilt full list with 2 entries, got %#v", entries)
	}
	if entries[0].Subject != "Second" || entries[1].Subject != "First" {
		t.Fatalf("unexpected entries after reindex: %#v", entries)
	}
}

func TestReadMboxMessagesBytesClonesBufferRecords(t *testing.T) {
	mbox := []byte(strings.Join([]string{
		"From one@example.com Tue Jul 14 10:00:00 2026",
		"Message-ID: <one@example.com>",
		"Subject: One",
		"",
		"short",
		"From two@example.com Tue Jul 14 10:01:00 2026",
		"Message-ID: <two@example.com>",
		"Subject: Two",
		"",
		"this message is deliberately much longer than the first one",
		"and has another line",
		"From three@example.com Tue Jul 14 10:02:00 2026",
		"Message-ID: <three@example.com>",
		"Subject: Three",
		"",
		"tiny",
		"",
	}, "\n"))

	msgs := readMboxMessagesBytes(mbox)
	if len(msgs) != 3 {
		t.Fatalf("expected 3 messages, got %d", len(msgs))
	}
	wants := []string{"Subject: One", "Subject: Two", "Subject: Three"}
	for i, want := range wants {
		if !bytes.Contains(msgs[i], []byte(want)) {
			t.Fatalf("message %d does not contain %q: %q", i, want, msgs[i])
		}
	}
	if bytes.Contains(msgs[0], []byte("Subject: Two")) || bytes.Contains(msgs[1], []byte("Subject: Three")) {
		t.Fatalf("messages share buffer contents: %#q", msgs)
	}
}

func TestPlainIndexedReadUsesOffsetAcrossMissingMiddleIndex(t *testing.T) {
	oldCfg, oldDB := Cfg, DB
	root := t.TempDir()
	t.Cleanup(func() {
		if DB != nil {
			_ = DB.Close()
		}
		Cfg, DB = oldCfg, oldDB
	})

	Cfg = Config{DBPath: filepath.Join(root, "mail-graveyard.db"), MboxRoot: filepath.Join(root, "backup")}
	if err := ConnectDB(true); err != nil {
		t.Fatal(err)
	}
	if err := SaveArchiveMailbox("archive"); err != nil {
		t.Fatal(err)
	}
	if err := SaveAccount(Account{
		Name: "test-account", SrcHost: "source.example", SrcPort: 993, SrcSecurity: "tls", SrcUser: "source@example.com", SrcPass: "x",
		DstHost: "target.example", DstPort: 993, DstSecurity: "tls", DstUser: "target@example.com", DstPass: "x", MboxDir: "archive", Active: true,
	}); err != nil {
		t.Fatal(err)
	}
	account, err := GetAccount("test-account")
	if err != nil {
		t.Fatal(err)
	}
	writer, err := NewMboxWriter(filepath.Join(Cfg.MboxRoot, "archive"))
	if err != nil {
		t.Fatal(err)
	}
	firstInfo, err := writer.Append("INBOX", testRawMessage("first@example.com", "First"))
	if err != nil {
		t.Fatal(err)
	}
	if err := SaveMboxIndex(MboxIndexEntry{AccountID: account.ID, Folder: "INBOX", MessageID: "first@example.com", Subject: "First", FileOffset: firstInfo.FileOffset, FrameLen: firstInfo.FrameLen, InnerOffset: firstInfo.InnerOffset, InnerLen: firstInfo.InnerLen}); err != nil {
		t.Fatal(err)
	}
	if _, err := writer.Append("INBOX", testRawMessage("missing@example.com", "Missing middle")); err != nil {
		t.Fatal(err)
	}
	thirdInfo, err := writer.Append("INBOX", testRawMessage("third@example.com", "Third"))
	if err != nil {
		t.Fatal(err)
	}
	if err := SaveMboxIndex(MboxIndexEntry{AccountID: account.ID, Folder: "INBOX", MessageID: "third@example.com", Subject: "Third", FileOffset: thirdInfo.FileOffset, FrameLen: thirdInfo.FrameLen, InnerOffset: thirdInfo.InnerOffset, InnerLen: thirdInfo.InnerLen}); err != nil {
		t.Fatal(err)
	}
	if err := UpdateMboxIndexState(account.ID, "INBOX", thirdInfo.FileOffset+thirdInfo.FrameLen); err != nil {
		t.Fatal(err)
	}

	raw, err := ReadMboxMessage(thirdInfo.Path, 1)
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Contains(raw, []byte("Subject: Third")) || bytes.Contains(raw, []byte("Missing middle")) {
		t.Fatalf("indexed read drifted to the file position: %q", raw)
	}

	if err := ReindexMboxFile(account.ID, "INBOX", thirdInfo.Path); err != nil {
		t.Fatal(err)
	}
	for i, subject := range []string{"First", "Missing middle", "Third"} {
		raw, err := ReadMboxMessage(thirdInfo.Path, i)
		if err != nil {
			t.Fatalf("rebuilt index %d: %v", i, err)
		}
		if !bytes.Contains(raw, []byte("Subject: "+subject)) {
			t.Fatalf("rebuilt index %d returned wrong message: %q", i, raw)
		}
	}
}

func TestConnectDBConfiguresBusyTimeoutAndCopyStages(t *testing.T) {
	oldCfg, oldDB := Cfg, DB
	root := t.TempDir()
	t.Cleanup(func() {
		if DB != nil {
			_ = DB.Close()
		}
		Cfg, DB = oldCfg, oldDB
	})
	Cfg = Config{DBPath: filepath.Join(root, "mail-graveyard.db"), MboxRoot: filepath.Join(root, "backup")}
	if err := ConnectDB(true); err != nil {
		t.Fatal(err)
	}
	var timeout int
	if err := DB.QueryRow(`PRAGMA busy_timeout`).Scan(&timeout); err != nil {
		t.Fatal(err)
	}
	if timeout != 10000 {
		t.Fatalf("busy_timeout=%d, want 10000", timeout)
	}
	if err := SaveAccount(Account{
		Name: "stage-account", SrcHost: "source.example", SrcPort: 993, SrcSecurity: "tls", SrcUser: "source@example.com", SrcPass: "x",
		DstHost: "target.example", DstPort: 993, DstSecurity: "tls", DstUser: "target@example.com", DstPass: "x", Active: true,
	}); err != nil {
		t.Fatal(err)
	}
	account, err := GetAccount("stage-account")
	if err != nil {
		t.Fatal(err)
	}
	if err := MarkMboxCopied(account.ID, "INBOX", "stage@example.com"); err != nil {
		t.Fatal(err)
	}
	state, err := GetCopyState(account.ID, "INBOX", "stage@example.com")
	if err != nil {
		t.Fatal(err)
	}
	if !state.MboxDone || state.TargetDone {
		t.Fatalf("unexpected mbox-only state: %#v", state)
	}
	if err := MarkTargetCopied(account.ID, "INBOX", "stage@example.com"); err != nil {
		t.Fatal(err)
	}
	state, err = GetCopyState(account.ID, "INBOX", "stage@example.com")
	if err != nil {
		t.Fatal(err)
	}
	if !state.MboxDone || !state.TargetDone {
		t.Fatalf("unexpected completed state: %#v", state)
	}
	first := MessageIdentity{MessageID: "reused@example.com", BodySHA256: bodySHA256([]byte("first"))}
	second := MessageIdentity{MessageID: "reused@example.com", BodySHA256: bodySHA256([]byte("second"))}
	if err := MarkIdentityCopied(account.ID, "INBOX", first); err != nil {
		t.Fatal(err)
	}
	state, err = GetCopyIdentityState(account.ID, "INBOX", second)
	if err != nil {
		t.Fatal(err)
	}
	if state.MboxDone || state.TargetDone {
		t.Fatalf("byte-different message with reused ID was hidden: %#v", state)
	}
}

func TestIdentitySchemaUpgradePreservesLegacyAliases(t *testing.T) {
	oldCfg, oldDB := Cfg, DB
	root := t.TempDir()
	path := filepath.Join(root, "legacy.db")
	rawDB, err := sql.Open("sqlite", path)
	if err != nil {
		t.Fatal(err)
	}
	legacySchema := []string{
		`CREATE TABLE accounts(id INTEGER PRIMARY KEY)`,
		`INSERT INTO accounts(id) VALUES(7)`,
		`CREATE TABLE copied(account_id INTEGER NOT NULL, folder TEXT NOT NULL, message_id TEXT NOT NULL, mbox_done INTEGER NOT NULL DEFAULT 1, target_done INTEGER NOT NULL DEFAULT 1, copied_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, UNIQUE(account_id, folder, message_id))`,
		`INSERT INTO copied(account_id, folder, message_id, mbox_done, target_done) VALUES(7, 'INBOX.Newbies ', 'sha256:legacy-raw', 1, 0)`,
		`CREATE TABLE mbox_index(account_id INTEGER NOT NULL, folder TEXT NOT NULL, seq INTEGER NOT NULL, message_id TEXT NOT NULL, subject TEXT NOT NULL DEFAULT '', from_addr TEXT NOT NULL DEFAULT '', date TEXT NOT NULL DEFAULT '', file_offset INTEGER NOT NULL, frame_len INTEGER NOT NULL, inner_offset INTEGER NOT NULL DEFAULT 0, inner_len INTEGER NOT NULL, UNIQUE(account_id, folder, message_id))`,
		`INSERT INTO mbox_index(account_id, folder, seq, message_id, file_offset, frame_len, inner_len) VALUES(7, 'INBOX.Newbies ', 0, 'sha256:legacy-raw', 0, 10, 10)`,
		`CREATE TABLE mbox_index_state(account_id INTEGER NOT NULL, folder TEXT NOT NULL, indexed_bytes INTEGER NOT NULL DEFAULT 0, UNIQUE(account_id, folder))`,
		`INSERT INTO mbox_index_state(account_id, folder, indexed_bytes) VALUES(7, 'INBOX.Newbies ', 10)`,
	}
	for _, stmt := range legacySchema {
		if _, err := rawDB.Exec(stmt); err != nil {
			_ = rawDB.Close()
			t.Fatal(err)
		}
	}
	if err := rawDB.Close(); err != nil {
		t.Fatal(err)
	}
	if DB != nil {
		_ = DB.Close()
	}
	t.Cleanup(func() {
		if DB != nil {
			_ = DB.Close()
		}
		Cfg, DB = oldCfg, oldDB
	})
	Cfg = Config{DBPath: path, MboxRoot: filepath.Join(root, "backup")}
	if err := ConnectDB(false); err != nil {
		t.Fatal(err)
	}
	var messageID, bodyHash string
	var mboxDone, targetDone int
	if err := DB.QueryRow(`SELECT message_id, body_sha256, mbox_done, target_done FROM copied WHERE account_id=7`).Scan(&messageID, &bodyHash, &mboxDone, &targetDone); err != nil {
		t.Fatal(err)
	}
	if messageID != "sha256:legacy-raw" || bodyHash != "" || mboxDone != 1 || targetDone != 0 {
		t.Fatalf("legacy alias changed during upgrade: id=%q hash=%q stages=%d/%d", messageID, bodyHash, mboxDone, targetDone)
	}
	var indexedBytes int64
	if err := DB.QueryRow(`SELECT indexed_bytes FROM mbox_index_state WHERE account_id=7 AND folder='INBOX.Newbies '`).Scan(&indexedBytes); err != nil {
		t.Fatal(err)
	}
	if indexedBytes != -1 {
		t.Fatalf("identity-less index was not invalidated: %d", indexedBytes)
	}
	stableHash := bodySHA256([]byte("archived body"))
	if err := ReplaceMboxIndex(7, "INBOX.Newbies", []MboxIndexEntry{{BodySHA256: stableHash, FrameLen: 10, InnerLen: 10}}, 10); err != nil {
		t.Fatal(err)
	}
	if err := DB.QueryRow(`SELECT mbox_done, target_done FROM copied WHERE account_id=7 AND folder='INBOX.Newbies' AND message_id='' AND body_sha256=?`, stableHash).Scan(&mboxDone, &targetDone); err != nil {
		t.Fatal(err)
	}
	if mboxDone != 1 || targetDone != 0 {
		t.Fatalf("reindex lost pending target state from legacy alias: %d/%d", mboxDone, targetDone)
	}
	if err := RemoveMboxIndexFolderAliases(7, map[string]bool{"INBOX.Newbies": true}); err != nil {
		t.Fatal(err)
	}
	var staleIndex, preservedAlias int
	if err := DB.QueryRow(`SELECT count(*) FROM mbox_index WHERE account_id=7 AND folder='INBOX.Newbies '`).Scan(&staleIndex); err != nil {
		t.Fatal(err)
	}
	if err := DB.QueryRow(`SELECT count(*) FROM copied WHERE account_id=7 AND folder='INBOX.Newbies ' AND message_id='sha256:legacy-raw'`).Scan(&preservedAlias); err != nil {
		t.Fatal(err)
	}
	if staleIndex != 0 || preservedAlias != 1 {
		t.Fatalf("folder alias cleanup stale_index=%d preserved_copy_alias=%d", staleIndex, preservedAlias)
	}
}

func TestHTMLToTextHandlesUnicodeBeforeCaseInsensitiveTags(t *testing.T) {
	tests := []struct {
		name string
		in   string
		want []string
		drop []string
	}{
		{
			name: "turkish dotted i",
			in:   "<p>Turkish İ test</p><BR>x",
			want: []string{"Turkish İ test", "x"},
		},
		{
			name: "kelvin sign",
			in:   "<DIV>Kelvin K test</DIV><Br/>x",
			want: []string{"Kelvin K test", "x"},
		},
		{
			name: "mixed blocks and repeated replacements",
			in:   "İ<style>bad K</STYLE><P>first</P>K<script>bad İ</SCRIPT><DIV>second</DIV><BR />third",
			want: []string{"İ", "first", "K", "second", "third"},
			drop: []string{"bad"},
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := htmlToText(tt.in)
			for _, want := range tt.want {
				if !strings.Contains(got, want) {
					t.Fatalf("htmlToText(%q) = %q, missing %q", tt.in, got, want)
				}
			}
			for _, drop := range tt.drop {
				if strings.Contains(got, drop) {
					t.Fatalf("htmlToText(%q) = %q, unexpectedly contains %q", tt.in, got, drop)
				}
			}
		})
	}
}

func TestASCIIFoldIndexReturnsOriginalByteOffset(t *testing.T) {
	for _, tt := range []struct {
		s      string
		needle string
		want   int
	}{
		{"İİ<BR>x", "<br>", len("İİ")},
		{"KK</DIV>", "</div>", len("KK")},
		{"prefix<Br />suffix", "<br />", len("prefix")},
	} {
		if got := asciiFoldIndex(tt.s, tt.needle); got != tt.want {
			t.Fatalf("asciiFoldIndex(%q, %q) = %d, want %d", tt.s, tt.needle, got, tt.want)
		}
	}
}

func TestMessageBodyPanicFallsBackToRawMessage(t *testing.T) {
	raw := []byte("Subject: Evidence\r\n\r\nraw body")
	got := renderMessageBodySafely(raw, "test account=archive folder=INBOX seq=7", func() string {
		panic("synthetic parser failure")
	})
	if !strings.Contains(got, "Darstellung fehlgeschlagen") || !strings.Contains(got, string(raw)) {
		t.Fatalf("panic fallback did not preserve raw message: %q", got)
	}
}

func testRawMessage(id, subject string) RawMessage {
	return RawMessage{
		MessageID:    id,
		InternalDate: time.Date(2026, 7, 14, 10, 11, 12, 0, time.UTC),
		Body: []byte("Message-ID: <" + id + ">\r\n" +
			"From: Sender <sender@example.com>\r\n" +
			"Subject: " + subject + "\r\n" +
			"Date: Tue, 14 Jul 2026 10:11:12 +0000\r\n" +
			"\r\n" +
			"Hello\r\n"),
	}
}