DonVoo/Mail-Graveyard
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use UID-stable target dedup scan

Browse source
This commit is contained in:
DonVoo 2026-07-14 21:17:45 +02:00
parent b16deffa25
commit 8e8c651eaa
3 changed files with 64 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

45
backend/04-imap-source.go
View file

@ -7,6 +7,7 @@ import (
"fmt"
"net"
"net/mail"
"sort"
"strconv"
"strings"
"time"
@ -259,6 +260,44 @@ func (m *imapClientMailbox) fetchHeaders(folder string, limit, offset int) ([]Me
return out, nil
}
func (m *imapClientMailbox) fetchAllHeadersByUID(folder string) ([]MessageHeader, error) {
selected, err := m.c.Select(folder, &imap.SelectOptions{ReadOnly: true}).Wait()
if err != nil {
return nil, err
}
if selected.NumMessages == 0 {
return nil, nil
}
uidSet := imap.UIDSet{}
uidSet.AddRange(1, 0)
cmd := m.c.Fetch(uidSet, &imap.FetchOptions{
Envelope: true,
Flags: true,
InternalDate: true,
RFC822Size: true,
UID: true,
})
defer cmd.Close()
var out []MessageHeader
for {
data := cmd.Next()
if data == nil {
break
}
buf, err := data.Collect()
if err != nil {
return out, err
}
out = append(out, messageHeaderFromFetch(buf))
}
if err := cmd.Close(); err != nil {
return out, err
}
sortMessageHeadersByUID(out)
return out, nil
}
func (m *imapClientMailbox) fetchOne(folder string, uid uint32) (RawMessage, error) {
if uid == 0 {
return RawMessage{}, fmt.Errorf("Nachricht ohne UID")
@ -418,6 +457,12 @@ func reverseHeaders(headers []MessageHeader) {
}
}
func sortMessageHeadersByUID(headers []MessageHeader) {
sort.SliceStable(headers, func(i, j int) bool {
return headers[i].UID < headers[j].UID
})
}
func messageIDFromFetch(buf *imapclient.FetchMessageBuffer, body []byte) string {
if buf != nil && buf.Envelope != nil && strings.TrimSpace(buf.Envelope.MessageID) != "" {
return normalizeMessageID(buf.Envelope.MessageID)

5
backend/05-imap-target.go
View file

@ -17,6 +17,7 @@ type TargetMailbox interface {
EnsureFolder(name string) error // anlegen + subscriben, falls fehlt
Append(folder string, m RawMessage) error // mit m.Flags und m.InternalDate
Headers(folder string, limit, offset int) ([]MessageHeader, error)
AllHeaders(folder string) ([]MessageHeader, error)
FetchOne(folder string, uid uint32) (RawMessage, error)
DeleteUIDs(folder string, uids []uint32) error
Close() error
@ -126,6 +127,10 @@ func (t *imapTarget) Headers(folder string, limit, offset int) ([]MessageHeader,
return t.mailbox.fetchHeaders(folder, limit, offset)
}
func (t *imapTarget) AllHeaders(folder string) ([]MessageHeader, error) {
return t.mailbox.fetchAllHeadersByUID(folder)
}
func (t *imapTarget) FetchOne(folder string, uid uint32) (RawMessage, error) {
return t.mailbox.fetchOne(folder, uid)
}

31
backend/13-target-dedup.go
View file

@ -13,6 +13,7 @@ type TargetDedupReport struct {
Folders int
Groups int
ExtraCopies int
Candidates int
WithoutID int
Deleted int
FolderReports []TargetDedupFolderReport
@ -58,6 +59,7 @@ func DedupTarget(name string, folders []string, apply bool) (TargetDedupReport,
report.Folders++
report.Groups += folderReport.Groups
report.ExtraCopies += folderReport.ExtraCopies
report.Candidates += len(folderReport.Deletes)
report.WithoutID += folderReport.WithoutID
if apply {
report.Deleted += len(folderReport.Deletes)
@ -65,29 +67,24 @@ func DedupTarget(name string, folders []string, apply bool) (TargetDedupReport,
report.FolderReports = append(report.FolderReports, folderReport)
}
log.Printf("target dedup %s apply=%v folders=%d groups=%d extra=%d candidates=%d deleted=%d without_id=%d",
report.Account, report.Apply, report.Folders, report.Groups, report.ExtraCopies, report.ExtraCopies, report.Deleted, report.WithoutID)
report.Account, report.Apply, report.Folders, report.Groups, report.ExtraCopies, report.Candidates, report.Deleted, report.WithoutID)
return report, nil
}
func dedupTargetFolder(dst TargetMailbox, folder string, apply bool) (TargetDedupFolderReport, error) {
report := TargetDedupFolderReport{Folder: folder}
byID := map[string][]uint32{}
for offset := 0; ; offset += mailboxListLimit {
headers, err := dst.Headers(folder, mailboxListLimit, offset)
if err != nil {
return report, fmt.Errorf("%s headers: %w", folder, err)
}
if len(headers) == 0 {
break
}
for _, header := range headers {
id := normalizeMessageID(header.MessageID)
if id == "" {
report.WithoutID++
continue
}
byID[id] = append(byID[id], header.UID)
headers, err := dst.AllHeaders(folder)
if err != nil {
return report, fmt.Errorf("%s headers: %w", folder, err)
}
for _, header := range headers {
id := normalizeMessageID(header.MessageID)
if id == "" {
report.WithoutID++
continue
}
byID[id] = append(byID[id], header.UID)
}
var deleteUIDs []uint32
for id, uids := range byID {
@ -129,7 +126,7 @@ func LogTargetDedupReport(report TargetDedupReport) {
mode = "APPLY"
}
log.Printf("target dedup report %s account=%s folders=%d groups=%d extra=%d candidates=%d deleted=%d without_id=%d",
mode, report.Account, report.Folders, report.Groups, report.ExtraCopies, report.ExtraCopies, report.Deleted, report.WithoutID)
mode, report.Account, report.Folders, report.Groups, report.ExtraCopies, report.Candidates, report.Deleted, report.WithoutID)
for _, folder := range report.FolderReports {
if folder.Groups == 0 && folder.WithoutID == 0 {
continue