Use UID-stable target dedup scan

backend/04-imap-source.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"net"
 	"net/mail"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -259,6 +260,44 @@ func (m *imapClientMailbox) fetchHeaders(folder string, limit, offset int) ([]Me
 	return out, nil
 }
 
+func (m *imapClientMailbox) fetchAllHeadersByUID(folder string) ([]MessageHeader, error) {
+	selected, err := m.c.Select(folder, &imap.SelectOptions{ReadOnly: true}).Wait()
+	if err != nil {
+		return nil, err
+	}
+	if selected.NumMessages == 0 {
+		return nil, nil
+	}
+	uidSet := imap.UIDSet{}
+	uidSet.AddRange(1, 0)
+	cmd := m.c.Fetch(uidSet, &imap.FetchOptions{
+		Envelope:     true,
+		Flags:        true,
+		InternalDate: true,
+		RFC822Size:   true,
+		UID:          true,
+	})
+	defer cmd.Close()
+
+	var out []MessageHeader
+	for {
+		data := cmd.Next()
+		if data == nil {
+			break
+		}
+		buf, err := data.Collect()
+		if err != nil {
+			return out, err
+		}
+		out = append(out, messageHeaderFromFetch(buf))
+	}
+	if err := cmd.Close(); err != nil {
+		return out, err
+	}
+	sortMessageHeadersByUID(out)
+	return out, nil
+}
+
 func (m *imapClientMailbox) fetchOne(folder string, uid uint32) (RawMessage, error) {
 	if uid == 0 {
 		return RawMessage{}, fmt.Errorf("Nachricht ohne UID")
@@ -418,6 +457,12 @@ func reverseHeaders(headers []MessageHeader) {
 	}
 }
 
+func sortMessageHeadersByUID(headers []MessageHeader) {
+	sort.SliceStable(headers, func(i, j int) bool {
+		return headers[i].UID < headers[j].UID
+	})
+}
+
 func messageIDFromFetch(buf *imapclient.FetchMessageBuffer, body []byte) string {
 	if buf != nil && buf.Envelope != nil && strings.TrimSpace(buf.Envelope.MessageID) != "" {
 		return normalizeMessageID(buf.Envelope.MessageID)

backend/05-imap-target.go

@@ -17,6 +17,7 @@ type TargetMailbox interface {
 	EnsureFolder(name string) error           // anlegen + subscriben, falls fehlt
 	Append(folder string, m RawMessage) error // mit m.Flags und m.InternalDate
 	Headers(folder string, limit, offset int) ([]MessageHeader, error)
+	AllHeaders(folder string) ([]MessageHeader, error)
 	FetchOne(folder string, uid uint32) (RawMessage, error)
 	DeleteUIDs(folder string, uids []uint32) error
 	Close() error
@@ -126,6 +127,10 @@ func (t *imapTarget) Headers(folder string, limit, offset int) ([]MessageHeader,
 	return t.mailbox.fetchHeaders(folder, limit, offset)
 }
 
+func (t *imapTarget) AllHeaders(folder string) ([]MessageHeader, error) {
+	return t.mailbox.fetchAllHeadersByUID(folder)
+}
+
 func (t *imapTarget) FetchOne(folder string, uid uint32) (RawMessage, error) {
 	return t.mailbox.fetchOne(folder, uid)
 }

backend/13-target-dedup.go

@@ -13,6 +13,7 @@ type TargetDedupReport struct {
 	Folders       int
 	Groups        int
 	ExtraCopies   int
+	Candidates    int
 	WithoutID     int
 	Deleted       int
 	FolderReports []TargetDedupFolderReport
@@ -58,6 +59,7 @@ func DedupTarget(name string, folders []string, apply bool) (TargetDedupReport,
 		report.Folders++
 		report.Groups += folderReport.Groups
 		report.ExtraCopies += folderReport.ExtraCopies
+		report.Candidates += len(folderReport.Deletes)
 		report.WithoutID += folderReport.WithoutID
 		if apply {
 			report.Deleted += len(folderReport.Deletes)
@@ -65,29 +67,24 @@ func DedupTarget(name string, folders []string, apply bool) (TargetDedupReport,
 		report.FolderReports = append(report.FolderReports, folderReport)
 	}
 	log.Printf("target dedup %s apply=%v folders=%d groups=%d extra=%d candidates=%d deleted=%d without_id=%d",
-		report.Account, report.Apply, report.Folders, report.Groups, report.ExtraCopies, report.ExtraCopies, report.Deleted, report.WithoutID)
+		report.Account, report.Apply, report.Folders, report.Groups, report.ExtraCopies, report.Candidates, report.Deleted, report.WithoutID)
 	return report, nil
 }
 
 func dedupTargetFolder(dst TargetMailbox, folder string, apply bool) (TargetDedupFolderReport, error) {
 	report := TargetDedupFolderReport{Folder: folder}
 	byID := map[string][]uint32{}
-	for offset := 0; ; offset += mailboxListLimit {
+	headers, err := dst.AllHeaders(folder)
-		headers, err := dst.Headers(folder, mailboxListLimit, offset)
+	if err != nil {
-		if err != nil {
+		return report, fmt.Errorf("%s headers: %w", folder, err)
-			return report, fmt.Errorf("%s headers: %w", folder, err)
+	}
-		}
+	for _, header := range headers {
-		if len(headers) == 0 {
+		id := normalizeMessageID(header.MessageID)
-			break
+		if id == "" {
-		}
+			report.WithoutID++
-		for _, header := range headers {
+			continue
-			id := normalizeMessageID(header.MessageID)
-			if id == "" {
-				report.WithoutID++
-				continue
-			}
-			byID[id] = append(byID[id], header.UID)
 		}
+		byID[id] = append(byID[id], header.UID)
 	}
 	var deleteUIDs []uint32
 	for id, uids := range byID {
@@ -129,7 +126,7 @@ func LogTargetDedupReport(report TargetDedupReport) {
 		mode = "APPLY"
 	}
 	log.Printf("target dedup report %s account=%s folders=%d groups=%d extra=%d candidates=%d deleted=%d without_id=%d",
-		mode, report.Account, report.Folders, report.Groups, report.ExtraCopies, report.ExtraCopies, report.Deleted, report.WithoutID)
+		mode, report.Account, report.Folders, report.Groups, report.ExtraCopies, report.Candidates, report.Deleted, report.WithoutID)
 	for _, folder := range report.FolderReports {
 		if folder.Groups == 0 && folder.WithoutID == 0 {
 			continue