diff --git a/backend/04-imap-source.go b/backend/04-imap-source.go index 0a48b99..88184fd 100644 --- a/backend/04-imap-source.go +++ b/backend/04-imap-source.go @@ -7,6 +7,7 @@ import ( "fmt" "net" "net/mail" + "sort" "strconv" "strings" "time" @@ -259,6 +260,44 @@ func (m *imapClientMailbox) fetchHeaders(folder string, limit, offset int) ([]Me return out, nil } +func (m *imapClientMailbox) fetchAllHeadersByUID(folder string) ([]MessageHeader, error) { + selected, err := m.c.Select(folder, &imap.SelectOptions{ReadOnly: true}).Wait() + if err != nil { + return nil, err + } + if selected.NumMessages == 0 { + return nil, nil + } + uidSet := imap.UIDSet{} + uidSet.AddRange(1, 0) + cmd := m.c.Fetch(uidSet, &imap.FetchOptions{ + Envelope: true, + Flags: true, + InternalDate: true, + RFC822Size: true, + UID: true, + }) + defer cmd.Close() + + var out []MessageHeader + for { + data := cmd.Next() + if data == nil { + break + } + buf, err := data.Collect() + if err != nil { + return out, err + } + out = append(out, messageHeaderFromFetch(buf)) + } + if err := cmd.Close(); err != nil { + return out, err + } + sortMessageHeadersByUID(out) + return out, nil +} + func (m *imapClientMailbox) fetchOne(folder string, uid uint32) (RawMessage, error) { if uid == 0 { return RawMessage{}, fmt.Errorf("Nachricht ohne UID") @@ -418,6 +457,12 @@ func reverseHeaders(headers []MessageHeader) { } } +func sortMessageHeadersByUID(headers []MessageHeader) { + sort.SliceStable(headers, func(i, j int) bool { + return headers[i].UID < headers[j].UID + }) +} + func messageIDFromFetch(buf *imapclient.FetchMessageBuffer, body []byte) string { if buf != nil && buf.Envelope != nil && strings.TrimSpace(buf.Envelope.MessageID) != "" { return normalizeMessageID(buf.Envelope.MessageID) diff --git a/backend/05-imap-target.go b/backend/05-imap-target.go index 7419da8..1059d48 100644 --- a/backend/05-imap-target.go +++ b/backend/05-imap-target.go @@ -17,6 +17,7 @@ type TargetMailbox interface { EnsureFolder(name string) error // anlegen + subscriben, falls fehlt Append(folder string, m RawMessage) error // mit m.Flags und m.InternalDate Headers(folder string, limit, offset int) ([]MessageHeader, error) + AllHeaders(folder string) ([]MessageHeader, error) FetchOne(folder string, uid uint32) (RawMessage, error) DeleteUIDs(folder string, uids []uint32) error Close() error @@ -126,6 +127,10 @@ func (t *imapTarget) Headers(folder string, limit, offset int) ([]MessageHeader, return t.mailbox.fetchHeaders(folder, limit, offset) } +func (t *imapTarget) AllHeaders(folder string) ([]MessageHeader, error) { + return t.mailbox.fetchAllHeadersByUID(folder) +} + func (t *imapTarget) FetchOne(folder string, uid uint32) (RawMessage, error) { return t.mailbox.fetchOne(folder, uid) } diff --git a/backend/13-target-dedup.go b/backend/13-target-dedup.go index 7686b27..b1e3999 100644 --- a/backend/13-target-dedup.go +++ b/backend/13-target-dedup.go @@ -13,6 +13,7 @@ type TargetDedupReport struct { Folders int Groups int ExtraCopies int + Candidates int WithoutID int Deleted int FolderReports []TargetDedupFolderReport @@ -58,6 +59,7 @@ func DedupTarget(name string, folders []string, apply bool) (TargetDedupReport, report.Folders++ report.Groups += folderReport.Groups report.ExtraCopies += folderReport.ExtraCopies + report.Candidates += len(folderReport.Deletes) report.WithoutID += folderReport.WithoutID if apply { report.Deleted += len(folderReport.Deletes) @@ -65,29 +67,24 @@ func DedupTarget(name string, folders []string, apply bool) (TargetDedupReport, report.FolderReports = append(report.FolderReports, folderReport) } log.Printf("target dedup %s apply=%v folders=%d groups=%d extra=%d candidates=%d deleted=%d without_id=%d", - report.Account, report.Apply, report.Folders, report.Groups, report.ExtraCopies, report.ExtraCopies, report.Deleted, report.WithoutID) + report.Account, report.Apply, report.Folders, report.Groups, report.ExtraCopies, report.Candidates, report.Deleted, report.WithoutID) return report, nil } func dedupTargetFolder(dst TargetMailbox, folder string, apply bool) (TargetDedupFolderReport, error) { report := TargetDedupFolderReport{Folder: folder} byID := map[string][]uint32{} - for offset := 0; ; offset += mailboxListLimit { - headers, err := dst.Headers(folder, mailboxListLimit, offset) - if err != nil { - return report, fmt.Errorf("%s headers: %w", folder, err) - } - if len(headers) == 0 { - break - } - for _, header := range headers { - id := normalizeMessageID(header.MessageID) - if id == "" { - report.WithoutID++ - continue - } - byID[id] = append(byID[id], header.UID) + headers, err := dst.AllHeaders(folder) + if err != nil { + return report, fmt.Errorf("%s headers: %w", folder, err) + } + for _, header := range headers { + id := normalizeMessageID(header.MessageID) + if id == "" { + report.WithoutID++ + continue } + byID[id] = append(byID[id], header.UID) } var deleteUIDs []uint32 for id, uids := range byID { @@ -129,7 +126,7 @@ func LogTargetDedupReport(report TargetDedupReport) { mode = "APPLY" } log.Printf("target dedup report %s account=%s folders=%d groups=%d extra=%d candidates=%d deleted=%d without_id=%d", - mode, report.Account, report.Folders, report.Groups, report.ExtraCopies, report.ExtraCopies, report.Deleted, report.WithoutID) + mode, report.Account, report.Folders, report.Groups, report.ExtraCopies, report.Candidates, report.Deleted, report.WithoutID) for _, folder := range report.FolderReports { if folder.Groups == 0 && folder.WithoutID == 0 { continue