Stabilize mail identity: canonical hash, index drift, HTML panic
Behebt drei Fehler derselben Klasse: an jeder Stelle wurde derselbe Wert zweimal berechnet, statt einmal berechnet und weitergereicht - und die beiden Berechnungen liefen auseinander. 1. Instabiler Ersatzschluessel (bug-hashkey-instabil.md) Mails ohne Message-ID bekamen sha256 ueber die IMAP-Rohbytes, der Reindex hashte dieselbe Mail ueber die mbox-gespeicherten Bytes (>From-Quoting, andere Zeilenenden) -> zwei Schluessel, 4.024 Doppel-Eintraege in copied. Fix: canonicalMessageBytes() bringt beide Seiten auf eine Form (>From zurueckdrehen, CRLF->LF, Trailing-Newlines weg). Alle Pfade (Index, Migration, Viewer, Dedup) nutzen dieselbe Funktion. body_sha256 in copied + mbox_index, UNIQUE erweitert. 2. Index-Drift (bug-index-drift.md) Der plain-mbox-Reader las nach Datei-Position statt nach dem gespeicherten file_offset. Weil SQLITE_BUSY (busy_timeout=0) je Ordner einen Index-Eintrag verschluckt hatte, war ab dieser Luecke alles um 1 verschoben: Klick auf Mail X zeigte Mail X+1. Betroffen genau die 6 Ordner mit Busy-Fehler beim Rettungslauf. Fix: busy_timeout=10000, Lesen ueber file_offset, --reindex --rebuild. 3. HTML-Vorschau-Panic (bug-htmltotext-panic.md) replaceCaseInsensitive/stripHTMLBlock indizierten mit Offsets aus strings.ToLower(s) in s - ToLower ist nicht byte-laengen-erhaltend (z.B. U+0130). ~0,1% der Mails brachten die Vorschau zum Absturz. Fix: asciiFoldIndex() sucht direkt auf den Original-Bytes; zusaetzlich Rohtext-Fallback, damit keine archivierte Mail unsichtbar wird. Weiter: Archiv-zuerst-Reihenfolge (mbox_done/target_done) - ein sterbendes Ziel kostet keine Archiv-Kopie mehr; Dedup vergleicht zusaetzlich den Inhalt und schuetzt byte-verschiedene Varianten. WICHTIG: Die 62.073 alten Alias-Zeilen in copied bleiben bewusst erhalten. Sie sehen wie Muell aus, sind aber die Zeilen, auf die der alte Schluessel matcht - ein Loeschen wuerde Mails erneut in fremde Postfaecher kopieren. Verifiziert: go test ./... gruen; 58.051 Archivmails, 0 ohne copied-Zeile; Drift 0 ueber alle 69 Ordner; 2.000 HTTP-Stichproben ueber 8 Ordner: 0 falsche Zuordnung, 0 Panics; Watcher 220 + 60 Mails archived=0 target=0 errors=0. Live als Image fix-identity2-20260716. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
DonVoo
parent
6c4073a01c
commit
9f5cc59af7
15 changed files with 1650 additions and 181 deletions
|
|
@ -15,21 +15,24 @@ type TargetDedupReport struct {
|
|||
ExtraCopies int
|
||||
Candidates int
|
||||
WithoutID int
|
||||
VariantGroups int
|
||||
Deleted int
|
||||
FolderReports []TargetDedupFolderReport
|
||||
}
|
||||
|
||||
type TargetDedupFolderReport struct {
|
||||
Folder string
|
||||
Groups int
|
||||
ExtraCopies int
|
||||
WithoutID int
|
||||
Deletes []TargetDedupDelete
|
||||
Folder string
|
||||
Groups int
|
||||
ExtraCopies int
|
||||
WithoutID int
|
||||
VariantGroups int
|
||||
Deletes []TargetDedupDelete
|
||||
}
|
||||
|
||||
type TargetDedupDelete struct {
|
||||
UID uint32
|
||||
MessageID string
|
||||
UID uint32
|
||||
MessageID string
|
||||
BodySHA256 string
|
||||
}
|
||||
|
||||
func DedupTarget(name string, folders []string, apply bool) (TargetDedupReport, error) {
|
||||
|
|
@ -61,13 +64,14 @@ func DedupTarget(name string, folders []string, apply bool) (TargetDedupReport,
|
|||
report.ExtraCopies += folderReport.ExtraCopies
|
||||
report.Candidates += len(folderReport.Deletes)
|
||||
report.WithoutID += folderReport.WithoutID
|
||||
report.VariantGroups += folderReport.VariantGroups
|
||||
if apply {
|
||||
report.Deleted += len(folderReport.Deletes)
|
||||
}
|
||||
report.FolderReports = append(report.FolderReports, folderReport)
|
||||
}
|
||||
log.Printf("target dedup %s apply=%v folders=%d groups=%d extra=%d candidates=%d deleted=%d without_id=%d",
|
||||
report.Account, report.Apply, report.Folders, report.Groups, report.ExtraCopies, report.Candidates, report.Deleted, report.WithoutID)
|
||||
log.Printf("target dedup %s apply=%v folders=%d groups=%d extra=%d candidates=%d deleted=%d without_id=%d protected_variant_groups=%d",
|
||||
report.Account, report.Apply, report.Folders, report.Groups, report.ExtraCopies, report.Candidates, report.Deleted, report.WithoutID, report.VariantGroups)
|
||||
return report, nil
|
||||
}
|
||||
|
||||
|
|
@ -91,12 +95,35 @@ func dedupTargetFolder(dst TargetMailbox, folder string, apply bool) (TargetDedu
|
|||
if len(uids) < 2 {
|
||||
continue
|
||||
}
|
||||
sort.Slice(uids, func(i, j int) bool { return uids[i] < uids[j] })
|
||||
report.Groups++
|
||||
report.ExtraCopies += len(uids) - 1
|
||||
for _, uid := range uids[1:] {
|
||||
report.Deletes = append(report.Deletes, TargetDedupDelete{UID: uid, MessageID: id})
|
||||
deleteUIDs = append(deleteUIDs, uid)
|
||||
byBody := map[string][]uint32{}
|
||||
for _, uid := range uids {
|
||||
msg, err := dst.FetchOne(folder, uid)
|
||||
if err != nil {
|
||||
return report, fmt.Errorf("%s fetch uid %d for safe dedup: %w", folder, uid, err)
|
||||
}
|
||||
hash := bodySHA256(msg.Body)
|
||||
byBody[hash] = append(byBody[hash], uid)
|
||||
}
|
||||
if len(byBody) > 1 {
|
||||
report.VariantGroups++
|
||||
}
|
||||
hashes := make([]string, 0, len(byBody))
|
||||
for hash := range byBody {
|
||||
hashes = append(hashes, hash)
|
||||
}
|
||||
sort.Strings(hashes)
|
||||
for _, hash := range hashes {
|
||||
bodyUIDs := byBody[hash]
|
||||
if len(bodyUIDs) < 2 {
|
||||
continue
|
||||
}
|
||||
sort.Slice(bodyUIDs, func(i, j int) bool { return bodyUIDs[i] < bodyUIDs[j] })
|
||||
report.Groups++
|
||||
report.ExtraCopies += len(bodyUIDs) - 1
|
||||
for _, uid := range bodyUIDs[1:] {
|
||||
report.Deletes = append(report.Deletes, TargetDedupDelete{UID: uid, MessageID: id, BodySHA256: hash})
|
||||
deleteUIDs = append(deleteUIDs, uid)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(report.Deletes, func(i, j int) bool {
|
||||
|
|
@ -108,14 +135,14 @@ func dedupTargetFolder(dst TargetMailbox, folder string, apply bool) (TargetDedu
|
|||
if apply && len(deleteUIDs) > 0 {
|
||||
sort.Slice(deleteUIDs, func(i, j int) bool { return deleteUIDs[i] < deleteUIDs[j] })
|
||||
for _, del := range report.Deletes {
|
||||
log.Printf("target dedup delete folder=%s uid=%d message_id=%s", folder, del.UID, del.MessageID)
|
||||
log.Printf("target dedup delete folder=%s uid=%d message_id=%s body_sha256=%s", folder, del.UID, del.MessageID, del.BodySHA256)
|
||||
}
|
||||
if err := dst.DeleteUIDs(folder, deleteUIDs); err != nil {
|
||||
return report, fmt.Errorf("%s delete uids: %w", folder, err)
|
||||
}
|
||||
}
|
||||
if report.Groups > 0 || report.WithoutID > 0 {
|
||||
log.Printf("target dedup folder=%s groups=%d extra=%d without_id=%d apply=%v", folder, report.Groups, report.ExtraCopies, report.WithoutID, apply)
|
||||
if report.Groups > 0 || report.WithoutID > 0 || report.VariantGroups > 0 {
|
||||
log.Printf("target dedup folder=%s groups=%d extra=%d without_id=%d protected_variant_groups=%d apply=%v", folder, report.Groups, report.ExtraCopies, report.WithoutID, report.VariantGroups, apply)
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
|
@ -125,13 +152,13 @@ func LogTargetDedupReport(report TargetDedupReport) {
|
|||
if report.Apply {
|
||||
mode = "APPLY"
|
||||
}
|
||||
log.Printf("target dedup report %s account=%s folders=%d groups=%d extra=%d candidates=%d deleted=%d without_id=%d",
|
||||
mode, report.Account, report.Folders, report.Groups, report.ExtraCopies, report.Candidates, report.Deleted, report.WithoutID)
|
||||
log.Printf("target dedup report %s account=%s folders=%d groups=%d extra=%d candidates=%d deleted=%d without_id=%d protected_variant_groups=%d",
|
||||
mode, report.Account, report.Folders, report.Groups, report.ExtraCopies, report.Candidates, report.Deleted, report.WithoutID, report.VariantGroups)
|
||||
for _, folder := range report.FolderReports {
|
||||
if folder.Groups == 0 && folder.WithoutID == 0 {
|
||||
if folder.Groups == 0 && folder.WithoutID == 0 && folder.VariantGroups == 0 {
|
||||
continue
|
||||
}
|
||||
log.Printf("target dedup report folder=%s groups=%d extra=%d without_id=%d", folder.Folder, folder.Groups, folder.ExtraCopies, folder.WithoutID)
|
||||
log.Printf("target dedup report folder=%s groups=%d extra=%d without_id=%d protected_variant_groups=%d", folder.Folder, folder.Groups, folder.ExtraCopies, folder.WithoutID, folder.VariantGroups)
|
||||
for _, del := range folder.Deletes {
|
||||
log.Printf("target dedup report candidate folder=%s uid=%d message_id=%s", folder.Folder, del.UID, del.MessageID)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue