DonVoo/Mail-Graveyard
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Stabilize mail identity: canonical hash, index drift, HTML panic

Browse source 
Behebt drei Fehler derselben Klasse: an jeder Stelle wurde derselbe Wert
zweimal berechnet, statt einmal berechnet und weitergereicht - und die
beiden Berechnungen liefen auseinander.

1. Instabiler Ersatzschluessel (bug-hashkey-instabil.md)
   Mails ohne Message-ID bekamen sha256 ueber die IMAP-Rohbytes, der
   Reindex hashte dieselbe Mail ueber die mbox-gespeicherten Bytes
   (>From-Quoting, andere Zeilenenden) -> zwei Schluessel, 4.024
   Doppel-Eintraege in copied.
   Fix: canonicalMessageBytes() bringt beide Seiten auf eine Form
   (>From zurueckdrehen, CRLF->LF, Trailing-Newlines weg). Alle Pfade
   (Index, Migration, Viewer, Dedup) nutzen dieselbe Funktion.
   body_sha256 in copied + mbox_index, UNIQUE erweitert.

2. Index-Drift (bug-index-drift.md)
   Der plain-mbox-Reader las nach Datei-Position statt nach dem
   gespeicherten file_offset. Weil SQLITE_BUSY (busy_timeout=0) je
   Ordner einen Index-Eintrag verschluckt hatte, war ab dieser Luecke
   alles um 1 verschoben: Klick auf Mail X zeigte Mail X+1.
   Betroffen genau die 6 Ordner mit Busy-Fehler beim Rettungslauf.
   Fix: busy_timeout=10000, Lesen ueber file_offset, --reindex --rebuild.

3. HTML-Vorschau-Panic (bug-htmltotext-panic.md)
   replaceCaseInsensitive/stripHTMLBlock indizierten mit Offsets aus
   strings.ToLower(s) in s - ToLower ist nicht byte-laengen-erhaltend
   (z.B. U+0130). ~0,1% der Mails brachten die Vorschau zum Absturz.
   Fix: asciiFoldIndex() sucht direkt auf den Original-Bytes; zusaetzlich
   Rohtext-Fallback, damit keine archivierte Mail unsichtbar wird.

Weiter: Archiv-zuerst-Reihenfolge (mbox_done/target_done) - ein
sterbendes Ziel kostet keine Archiv-Kopie mehr; Dedup vergleicht
zusaetzlich den Inhalt und schuetzt byte-verschiedene Varianten.

WICHTIG: Die 62.073 alten Alias-Zeilen in copied bleiben bewusst
erhalten. Sie sehen wie Muell aus, sind aber die Zeilen, auf die der
alte Schluessel matcht - ein Loeschen wuerde Mails erneut in fremde
Postfaecher kopieren.

Verifiziert: go test ./... gruen; 58.051 Archivmails, 0 ohne
copied-Zeile; Drift 0 ueber alle 69 Ordner; 2.000 HTTP-Stichproben
ueber 8 Ordner: 0 falsche Zuordnung, 0 Panics; Watcher 220 + 60 Mails
archived=0 target=0 errors=0. Live als Image fix-identity2-20260716.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
DonVoo 2026-07-16 13:22:27 +02:00
parent 6c4073a01c
commit 9f5cc59af7
15 changed files with 1650 additions and 181 deletions
Download patch file Download diff file Expand all files Collapse all files

181
backend/06-mbox.go
View file

@ -6,6 +6,7 @@ import (
"fmt"
"html"
"io"
"log"
"mime"
"mime/multipart"
"mime/quotedprintable"
@ -118,7 +119,7 @@ func mboxAppendInfoFromMessage(m RawMessage, record []byte) MboxAppendInfo {
parts := splitMboxRecordsWithOffsets(record)
if len(parts) > 0 {
info.InnerOffset = parts[0].Offset
info.InnerLen = int64(len(parts[0].Message))
info.InnerLen = int64(parts[0].Length)
}
msg, err := mail.ReadMessage(bytes.NewReader(m.Body))
if err != nil {
@ -274,6 +275,7 @@ func ReindexArchives(name string) error {
func reindexAccountArchive(account Account) error {
dir := filepath.Join(Cfg.MboxRoot, accountMboxDir(account))
canonicalFolders := map[string]bool{}
for _, path := range archiveMboxPaths(dir) {
folder := folderNameFromMboxPath(path)
if folder == "" {
@ -282,8 +284,9 @@ func reindexAccountArchive(account Account) error {
if err := ReindexMboxFile(account.ID, folder, path); err != nil {
return fmt.Errorf("%s %s: %w", account.Name, folder, err)
}
canonicalFolders[folder] = true
}
return nil
return RemoveMboxIndexFolderAliases(account.ID, canonicalFolders)
}
func reindexPlainMbox(path string) ([]MboxIndexEntry, error) {
@ -294,7 +297,7 @@ func reindexPlainMbox(path string) ([]MboxIndexEntry, error) {
var out []MboxIndexEntry
for _, part := range splitMboxRecordsWithOffsets(b) {
entry := indexEntryFromMboxRecord(part.Message, part.Offset, int64(part.Length), 0, int64(part.Length))
if entry.MessageID != "" {
if entry.MessageID != "" || entry.BodySHA256 != "" {
out = append(out, entry)
}
}
@ -318,8 +321,8 @@ func reindexZstdMbox(path string) ([]MboxIndexEntry, error) {
return nil, err
}
for _, part := range splitMboxRecordsWithOffsets(record) {
entry := indexEntryFromMboxRecord(part.Message, frame.Offset, int64(frame.Length), part.Offset, int64(len(part.Message)))
if entry.MessageID != "" {
entry := indexEntryFromMboxRecord(part.Message, frame.Offset, int64(frame.Length), part.Offset, int64(part.Length))
if entry.MessageID != "" || entry.BodySHA256 != "" {
out = append(out, entry)
}
}
@ -334,39 +337,44 @@ type mboxRecordPart struct {
}
func splitMboxRecordsWithOffsets(b []byte) []mboxRecordPart {
normalized := bytes.ReplaceAll(b, []byte("\r\n"), []byte("\n"))
lines := bytes.SplitAfter(normalized, []byte("\n"))
var out []mboxRecordPart
var cur bytes.Buffer
var curStart int64
var pos int64
inMsg := false
for _, lineWithNL := range lines {
line := bytes.TrimSuffix(lineWithNL, []byte("\n"))
if bytes.HasPrefix(line, []byte("From ")) {
if inMsg && cur.Len() > 0 {
msg := bytes.TrimRight(cur.Bytes(), "\n")
out = append(out, mboxRecordPart{Offset: curStart, Length: int(pos - curStart), Message: append([]byte(nil), msg...)})
cur.Reset()
}
inMsg = true
pos += int64(len(lineWithNL))
curStart = pos
continue
messageStart := -1
appendRecord := func(end int) {
if messageStart < 0 || end <= messageStart {
return
}
if inMsg {
if bytes.HasPrefix(line, []byte(">From ")) {
line = line[1:]
}
_, _ = cur.Write(line)
_ = cur.WriteByte('\n')
raw := b[messageStart:end]
message := unescapeMboxMessage(raw)
if len(message) == 0 {
return
}
pos += int64(len(lineWithNL))
out = append(out, mboxRecordPart{
Offset: int64(messageStart),
Length: end - messageStart,
Message: append([]byte(nil), message...),
})
}
if inMsg && cur.Len() > 0 {
msg := bytes.TrimRight(cur.Bytes(), "\n")
out = append(out, mboxRecordPart{Offset: curStart, Length: int(pos - curStart), Message: append([]byte(nil), msg...)})
for lineStart := 0; lineStart < len(b); {
relNL := bytes.IndexByte(b[lineStart:], '\n')
lineEnd, nextLine := len(b), len(b)
if relNL >= 0 {
lineEnd = lineStart + relNL
nextLine = lineEnd + 1
}
contentEnd := lineEnd
if contentEnd > lineStart && b[contentEnd-1] == '\r' {
contentEnd--
}
if bytes.HasPrefix(b[lineStart:contentEnd], []byte("From ")) {
appendRecord(lineStart)
messageStart = nextLine
}
if nextLine >= len(b) {
break
}
lineStart = nextLine
}
appendRecord(len(b))
return out
}
@ -399,15 +407,12 @@ func splitZstdFrames(b []byte) []zstdFramePart {
}
func indexEntryFromMboxRecord(raw []byte, fileOffset, frameLen, innerOffset, innerLen int64) MboxIndexEntry {
entry := MboxIndexEntry{FileOffset: fileOffset, FrameLen: frameLen, InnerOffset: innerOffset, InnerLen: innerLen}
entry := MboxIndexEntry{BodySHA256: bodySHA256(raw), FileOffset: fileOffset, FrameLen: frameLen, InnerOffset: innerOffset, InnerLen: innerLen}
msg, err := mail.ReadMessage(bytes.NewReader(raw))
if err != nil {
return entry
}
entry.MessageID = normalizeMessageID(msg.Header.Get("Message-ID"))
if entry.MessageID == "" {
entry.MessageID = messageID(raw)
}
entry.Subject = decodeHeader(msg.Header.Get("Subject"))
entry.From = decodeHeader(msg.Header.Get("From"))
entry.Date = decodeHeader(msg.Header.Get("Date"))
@ -415,19 +420,7 @@ func indexEntryFromMboxRecord(raw []byte, fileOffset, frameLen, innerOffset, inn
}
func unescapeMboxMessage(raw []byte) []byte {
raw = bytes.ReplaceAll(raw, []byte("\r\n"), []byte("\n"))
lines := bytes.Split(raw, []byte("\n"))
var b bytes.Buffer
for i, line := range lines {
if i > 0 {
_ = b.WriteByte('\n')
}
if bytes.HasPrefix(line, []byte(">From ")) {
line = line[1:]
}
_, _ = b.Write(line)
}
return bytes.TrimRight(b.Bytes(), "\n")
return canonicalMessageBytes(raw)
}
func readMboxMessagesBytes(b []byte) [][]byte {
@ -522,9 +515,6 @@ func readMboxMessageFromIndex(path string, index int) ([]byte, bool, error) {
if err != nil {
return nil, false, nil
}
if !strings.HasSuffix(strings.ToLower(path), ".zst") {
return nil, false, nil
}
f, err := os.Open(path)
if err != nil {
return nil, true, err
@ -533,10 +523,28 @@ func readMboxMessageFromIndex(path string, index int) ([]byte, bool, error) {
if _, err := f.Seek(entry.FileOffset, io.SeekStart); err != nil {
return nil, true, err
}
if entry.FrameLen <= 0 || entry.FrameLen > int64(^uint(0)>>1) {
return nil, true, fmt.Errorf("invalid indexed mbox length %d", entry.FrameLen)
}
frame := make([]byte, entry.FrameLen)
if _, err := io.ReadFull(f, frame); err != nil {
return nil, true, err
}
if !strings.HasSuffix(strings.ToLower(path), ".zst") {
// Entries written during migration point at the complete mbox record
// (separator included, InnerOffset > 0). Entries produced by a full
// rebuild point directly at the raw message bytes (InnerOffset == 0).
// Both variants are read by their persisted byte range, never by the
// message's positional number in a freshly split file.
if entry.InnerOffset > 0 {
messages := readMboxMessagesBytes(frame)
if len(messages) != 1 {
return nil, true, fmt.Errorf("invalid indexed mbox record at offset %d", entry.FileOffset)
}
return messages[0], true, nil
}
return unescapeMboxMessage(frame), true, nil
}
dec, err := zstd.NewReader(nil)
if err != nil {
return nil, true, err
@ -605,6 +613,29 @@ func decodeHeader(v string) string {
}
func messageBody(raw []byte) string {
return messageBodyWithContext(raw, "")
}
func messageBodyWithContext(raw []byte, context string) string {
return renderMessageBodySafely(raw, context, func() string {
return messageBodyUnsafe(raw)
})
}
func renderMessageBodySafely(raw []byte, context string, render func() string) (body string) {
defer func() {
if recovered := recover(); recovered != nil {
if strings.TrimSpace(context) == "" {
context = "unbekannter Vorschaupfad"
}
log.Printf("mail preview rendering panic (%s): %v", context, recovered)
body = "Darstellung fehlgeschlagen; Rohfassung:\n\n" + string(raw)
}
}()
return render()
}
func messageBodyUnsafe(raw []byte) string {
msg, err := mail.ReadMessage(bytes.NewReader(raw))
if err != nil {
return string(raw)
@ -738,11 +769,12 @@ func markHTMLBreaks(s string) string {
}
func replaceCaseInsensitive(s, old, new string) string {
if old == "" {
return s
}
var b strings.Builder
lower := strings.ToLower(s)
needle := strings.ToLower(old)
for {
i := strings.Index(lower, needle)
i := asciiFoldIndex(s, old)
if i < 0 {
b.WriteString(s)
return b.String()
@ -751,29 +783,56 @@ func replaceCaseInsensitive(s, old, new string) string {
b.WriteString(new)
cut := i + len(old)
s = s[cut:]
lower = lower[cut:]
}
}
func stripHTMLBlock(s, tag string) string {
lower := strings.ToLower(s)
open := "<" + tag
close := "</" + tag + ">"
for {
start := strings.Index(lower, open)
start := asciiFoldIndex(s, open)
if start < 0 {
return s
}
end := strings.Index(lower[start:], close)
end := asciiFoldIndex(s[start:], close)
if end < 0 {
return s[:start]
}
end += start + len(close)
s = s[:start] + " " + s[end:]
lower = strings.ToLower(s)
}
}
// asciiFoldIndex searches an ASCII needle case-insensitively in s. It scans
// the original bytes, so the returned index is always valid for slicing s even
// when s contains Unicode characters whose lowercase form has another byte
// length (for example Turkish dotted I or the Kelvin sign).
func asciiFoldIndex(s, needle string) int {
if needle == "" || len(s) < len(needle) {
return -1
}
for i := 0; i+len(needle) <= len(s); i++ {
matched := true
for j := 0; j < len(needle); j++ {
if asciiLower(s[i+j]) != asciiLower(needle[j]) {
matched = false
break
}
}
if matched {
return i
}
}
return -1
}
func asciiLower(c byte) byte {
if c >= 'A' && c <= 'Z' {
return c + ('a' - 'A')
}
return c
}
func decodeTransfer(body []byte, enc string) []byte {
switch strings.ToLower(strings.TrimSpace(enc)) {
case "quoted-printable":