Stabilize mail identity: canonical hash, index drift, HTML panic
Behebt drei Fehler derselben Klasse: an jeder Stelle wurde derselbe Wert zweimal berechnet, statt einmal berechnet und weitergereicht - und die beiden Berechnungen liefen auseinander. 1. Instabiler Ersatzschluessel (bug-hashkey-instabil.md) Mails ohne Message-ID bekamen sha256 ueber die IMAP-Rohbytes, der Reindex hashte dieselbe Mail ueber die mbox-gespeicherten Bytes (>From-Quoting, andere Zeilenenden) -> zwei Schluessel, 4.024 Doppel-Eintraege in copied. Fix: canonicalMessageBytes() bringt beide Seiten auf eine Form (>From zurueckdrehen, CRLF->LF, Trailing-Newlines weg). Alle Pfade (Index, Migration, Viewer, Dedup) nutzen dieselbe Funktion. body_sha256 in copied + mbox_index, UNIQUE erweitert. 2. Index-Drift (bug-index-drift.md) Der plain-mbox-Reader las nach Datei-Position statt nach dem gespeicherten file_offset. Weil SQLITE_BUSY (busy_timeout=0) je Ordner einen Index-Eintrag verschluckt hatte, war ab dieser Luecke alles um 1 verschoben: Klick auf Mail X zeigte Mail X+1. Betroffen genau die 6 Ordner mit Busy-Fehler beim Rettungslauf. Fix: busy_timeout=10000, Lesen ueber file_offset, --reindex --rebuild. 3. HTML-Vorschau-Panic (bug-htmltotext-panic.md) replaceCaseInsensitive/stripHTMLBlock indizierten mit Offsets aus strings.ToLower(s) in s - ToLower ist nicht byte-laengen-erhaltend (z.B. U+0130). ~0,1% der Mails brachten die Vorschau zum Absturz. Fix: asciiFoldIndex() sucht direkt auf den Original-Bytes; zusaetzlich Rohtext-Fallback, damit keine archivierte Mail unsichtbar wird. Weiter: Archiv-zuerst-Reihenfolge (mbox_done/target_done) - ein sterbendes Ziel kostet keine Archiv-Kopie mehr; Dedup vergleicht zusaetzlich den Inhalt und schuetzt byte-verschiedene Varianten. WICHTIG: Die 62.073 alten Alias-Zeilen in copied bleiben bewusst erhalten. Sie sehen wie Muell aus, sind aber die Zeilen, auf die der alte Schluessel matcht - ein Loeschen wuerde Mails erneut in fremde Postfaecher kopieren. Verifiziert: go test ./... gruen; 58.051 Archivmails, 0 ohne copied-Zeile; Drift 0 ueber alle 69 Ordner; 2.000 HTTP-Stichproben ueber 8 Ordner: 0 falsche Zuordnung, 0 Panics; Watcher 220 + 60 Mails archived=0 target=0 errors=0. Live als Image fix-identity2-20260716. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
DonVoo
parent
6c4073a01c
commit
9f5cc59af7
15 changed files with 1650 additions and 181 deletions
|
|
@ -2,6 +2,7 @@ package backend
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"database/sql"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
|
@ -240,6 +241,291 @@ func TestReadMboxMessagesBytesClonesBufferRecords(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestPlainIndexedReadUsesOffsetAcrossMissingMiddleIndex(t *testing.T) {
|
||||
oldCfg, oldDB := Cfg, DB
|
||||
root := t.TempDir()
|
||||
t.Cleanup(func() {
|
||||
if DB != nil {
|
||||
_ = DB.Close()
|
||||
}
|
||||
Cfg, DB = oldCfg, oldDB
|
||||
})
|
||||
|
||||
Cfg = Config{DBPath: filepath.Join(root, "mail-graveyard.db"), MboxRoot: filepath.Join(root, "backup")}
|
||||
if err := ConnectDB(true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SaveArchiveMailbox("archive"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SaveAccount(Account{
|
||||
Name: "test-account", SrcHost: "source.example", SrcPort: 993, SrcSecurity: "tls", SrcUser: "source@example.com", SrcPass: "x",
|
||||
DstHost: "target.example", DstPort: 993, DstSecurity: "tls", DstUser: "target@example.com", DstPass: "x", MboxDir: "archive", Active: true,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
account, err := GetAccount("test-account")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writer, err := NewMboxWriter(filepath.Join(Cfg.MboxRoot, "archive"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
firstInfo, err := writer.Append("INBOX", testRawMessage("first@example.com", "First"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SaveMboxIndex(MboxIndexEntry{AccountID: account.ID, Folder: "INBOX", MessageID: "first@example.com", Subject: "First", FileOffset: firstInfo.FileOffset, FrameLen: firstInfo.FrameLen, InnerOffset: firstInfo.InnerOffset, InnerLen: firstInfo.InnerLen}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := writer.Append("INBOX", testRawMessage("missing@example.com", "Missing middle")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
thirdInfo, err := writer.Append("INBOX", testRawMessage("third@example.com", "Third"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SaveMboxIndex(MboxIndexEntry{AccountID: account.ID, Folder: "INBOX", MessageID: "third@example.com", Subject: "Third", FileOffset: thirdInfo.FileOffset, FrameLen: thirdInfo.FrameLen, InnerOffset: thirdInfo.InnerOffset, InnerLen: thirdInfo.InnerLen}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := UpdateMboxIndexState(account.ID, "INBOX", thirdInfo.FileOffset+thirdInfo.FrameLen); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
raw, err := ReadMboxMessage(thirdInfo.Path, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Contains(raw, []byte("Subject: Third")) || bytes.Contains(raw, []byte("Missing middle")) {
|
||||
t.Fatalf("indexed read drifted to the file position: %q", raw)
|
||||
}
|
||||
|
||||
if err := ReindexMboxFile(account.ID, "INBOX", thirdInfo.Path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, subject := range []string{"First", "Missing middle", "Third"} {
|
||||
raw, err := ReadMboxMessage(thirdInfo.Path, i)
|
||||
if err != nil {
|
||||
t.Fatalf("rebuilt index %d: %v", i, err)
|
||||
}
|
||||
if !bytes.Contains(raw, []byte("Subject: "+subject)) {
|
||||
t.Fatalf("rebuilt index %d returned wrong message: %q", i, raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnectDBConfiguresBusyTimeoutAndCopyStages(t *testing.T) {
|
||||
oldCfg, oldDB := Cfg, DB
|
||||
root := t.TempDir()
|
||||
t.Cleanup(func() {
|
||||
if DB != nil {
|
||||
_ = DB.Close()
|
||||
}
|
||||
Cfg, DB = oldCfg, oldDB
|
||||
})
|
||||
Cfg = Config{DBPath: filepath.Join(root, "mail-graveyard.db"), MboxRoot: filepath.Join(root, "backup")}
|
||||
if err := ConnectDB(true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var timeout int
|
||||
if err := DB.QueryRow(`PRAGMA busy_timeout`).Scan(&timeout); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if timeout != 10000 {
|
||||
t.Fatalf("busy_timeout=%d, want 10000", timeout)
|
||||
}
|
||||
if err := SaveAccount(Account{
|
||||
Name: "stage-account", SrcHost: "source.example", SrcPort: 993, SrcSecurity: "tls", SrcUser: "source@example.com", SrcPass: "x",
|
||||
DstHost: "target.example", DstPort: 993, DstSecurity: "tls", DstUser: "target@example.com", DstPass: "x", Active: true,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
account, err := GetAccount("stage-account")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := MarkMboxCopied(account.ID, "INBOX", "stage@example.com"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state, err := GetCopyState(account.ID, "INBOX", "stage@example.com")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !state.MboxDone || state.TargetDone {
|
||||
t.Fatalf("unexpected mbox-only state: %#v", state)
|
||||
}
|
||||
if err := MarkTargetCopied(account.ID, "INBOX", "stage@example.com"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state, err = GetCopyState(account.ID, "INBOX", "stage@example.com")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !state.MboxDone || !state.TargetDone {
|
||||
t.Fatalf("unexpected completed state: %#v", state)
|
||||
}
|
||||
first := MessageIdentity{MessageID: "reused@example.com", BodySHA256: bodySHA256([]byte("first"))}
|
||||
second := MessageIdentity{MessageID: "reused@example.com", BodySHA256: bodySHA256([]byte("second"))}
|
||||
if err := MarkIdentityCopied(account.ID, "INBOX", first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state, err = GetCopyIdentityState(account.ID, "INBOX", second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state.MboxDone || state.TargetDone {
|
||||
t.Fatalf("byte-different message with reused ID was hidden: %#v", state)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIdentitySchemaUpgradePreservesLegacyAliases(t *testing.T) {
|
||||
oldCfg, oldDB := Cfg, DB
|
||||
root := t.TempDir()
|
||||
path := filepath.Join(root, "legacy.db")
|
||||
rawDB, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
legacySchema := []string{
|
||||
`CREATE TABLE accounts(id INTEGER PRIMARY KEY)`,
|
||||
`INSERT INTO accounts(id) VALUES(7)`,
|
||||
`CREATE TABLE copied(account_id INTEGER NOT NULL, folder TEXT NOT NULL, message_id TEXT NOT NULL, mbox_done INTEGER NOT NULL DEFAULT 1, target_done INTEGER NOT NULL DEFAULT 1, copied_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, UNIQUE(account_id, folder, message_id))`,
|
||||
`INSERT INTO copied(account_id, folder, message_id, mbox_done, target_done) VALUES(7, 'INBOX.Newbies ', 'sha256:legacy-raw', 1, 0)`,
|
||||
`CREATE TABLE mbox_index(account_id INTEGER NOT NULL, folder TEXT NOT NULL, seq INTEGER NOT NULL, message_id TEXT NOT NULL, subject TEXT NOT NULL DEFAULT '', from_addr TEXT NOT NULL DEFAULT '', date TEXT NOT NULL DEFAULT '', file_offset INTEGER NOT NULL, frame_len INTEGER NOT NULL, inner_offset INTEGER NOT NULL DEFAULT 0, inner_len INTEGER NOT NULL, UNIQUE(account_id, folder, message_id))`,
|
||||
`INSERT INTO mbox_index(account_id, folder, seq, message_id, file_offset, frame_len, inner_len) VALUES(7, 'INBOX.Newbies ', 0, 'sha256:legacy-raw', 0, 10, 10)`,
|
||||
`CREATE TABLE mbox_index_state(account_id INTEGER NOT NULL, folder TEXT NOT NULL, indexed_bytes INTEGER NOT NULL DEFAULT 0, UNIQUE(account_id, folder))`,
|
||||
`INSERT INTO mbox_index_state(account_id, folder, indexed_bytes) VALUES(7, 'INBOX.Newbies ', 10)`,
|
||||
}
|
||||
for _, stmt := range legacySchema {
|
||||
if _, err := rawDB.Exec(stmt); err != nil {
|
||||
_ = rawDB.Close()
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := rawDB.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if DB != nil {
|
||||
_ = DB.Close()
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if DB != nil {
|
||||
_ = DB.Close()
|
||||
}
|
||||
Cfg, DB = oldCfg, oldDB
|
||||
})
|
||||
Cfg = Config{DBPath: path, MboxRoot: filepath.Join(root, "backup")}
|
||||
if err := ConnectDB(false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var messageID, bodyHash string
|
||||
var mboxDone, targetDone int
|
||||
if err := DB.QueryRow(`SELECT message_id, body_sha256, mbox_done, target_done FROM copied WHERE account_id=7`).Scan(&messageID, &bodyHash, &mboxDone, &targetDone); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if messageID != "sha256:legacy-raw" || bodyHash != "" || mboxDone != 1 || targetDone != 0 {
|
||||
t.Fatalf("legacy alias changed during upgrade: id=%q hash=%q stages=%d/%d", messageID, bodyHash, mboxDone, targetDone)
|
||||
}
|
||||
var indexedBytes int64
|
||||
if err := DB.QueryRow(`SELECT indexed_bytes FROM mbox_index_state WHERE account_id=7 AND folder='INBOX.Newbies '`).Scan(&indexedBytes); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if indexedBytes != -1 {
|
||||
t.Fatalf("identity-less index was not invalidated: %d", indexedBytes)
|
||||
}
|
||||
stableHash := bodySHA256([]byte("archived body"))
|
||||
if err := ReplaceMboxIndex(7, "INBOX.Newbies", []MboxIndexEntry{{BodySHA256: stableHash, FrameLen: 10, InnerLen: 10}}, 10); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := DB.QueryRow(`SELECT mbox_done, target_done FROM copied WHERE account_id=7 AND folder='INBOX.Newbies' AND message_id='' AND body_sha256=?`, stableHash).Scan(&mboxDone, &targetDone); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if mboxDone != 1 || targetDone != 0 {
|
||||
t.Fatalf("reindex lost pending target state from legacy alias: %d/%d", mboxDone, targetDone)
|
||||
}
|
||||
if err := RemoveMboxIndexFolderAliases(7, map[string]bool{"INBOX.Newbies": true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var staleIndex, preservedAlias int
|
||||
if err := DB.QueryRow(`SELECT count(*) FROM mbox_index WHERE account_id=7 AND folder='INBOX.Newbies '`).Scan(&staleIndex); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := DB.QueryRow(`SELECT count(*) FROM copied WHERE account_id=7 AND folder='INBOX.Newbies ' AND message_id='sha256:legacy-raw'`).Scan(&preservedAlias); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if staleIndex != 0 || preservedAlias != 1 {
|
||||
t.Fatalf("folder alias cleanup stale_index=%d preserved_copy_alias=%d", staleIndex, preservedAlias)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTMLToTextHandlesUnicodeBeforeCaseInsensitiveTags(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in string
|
||||
want []string
|
||||
drop []string
|
||||
}{
|
||||
{
|
||||
name: "turkish dotted i",
|
||||
in: "<p>Turkish İ test</p><BR>x",
|
||||
want: []string{"Turkish İ test", "x"},
|
||||
},
|
||||
{
|
||||
name: "kelvin sign",
|
||||
in: "<DIV>Kelvin K test</DIV><Br/>x",
|
||||
want: []string{"Kelvin K test", "x"},
|
||||
},
|
||||
{
|
||||
name: "mixed blocks and repeated replacements",
|
||||
in: "İ<style>bad K</STYLE><P>first</P>K<script>bad İ</SCRIPT><DIV>second</DIV><BR />third",
|
||||
want: []string{"İ", "first", "K", "second", "third"},
|
||||
drop: []string{"bad"},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := htmlToText(tt.in)
|
||||
for _, want := range tt.want {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Fatalf("htmlToText(%q) = %q, missing %q", tt.in, got, want)
|
||||
}
|
||||
}
|
||||
for _, drop := range tt.drop {
|
||||
if strings.Contains(got, drop) {
|
||||
t.Fatalf("htmlToText(%q) = %q, unexpectedly contains %q", tt.in, got, drop)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestASCIIFoldIndexReturnsOriginalByteOffset(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
s string
|
||||
needle string
|
||||
want int
|
||||
}{
|
||||
{"İİ<BR>x", "<br>", len("İİ")},
|
||||
{"KK</DIV>", "</div>", len("KK")},
|
||||
{"prefix<Br />suffix", "<br />", len("prefix")},
|
||||
} {
|
||||
if got := asciiFoldIndex(tt.s, tt.needle); got != tt.want {
|
||||
t.Fatalf("asciiFoldIndex(%q, %q) = %d, want %d", tt.s, tt.needle, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMessageBodyPanicFallsBackToRawMessage(t *testing.T) {
|
||||
raw := []byte("Subject: Evidence\r\n\r\nraw body")
|
||||
got := renderMessageBodySafely(raw, "test account=archive folder=INBOX seq=7", func() string {
|
||||
panic("synthetic parser failure")
|
||||
})
|
||||
if !strings.Contains(got, "Darstellung fehlgeschlagen") || !strings.Contains(got, string(raw)) {
|
||||
t.Fatalf("panic fallback did not preserve raw message: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func testRawMessage(id, subject string) RawMessage {
|
||||
return RawMessage{
|
||||
MessageID: id,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue