Fix archive reindex and add target dedup

backend/13-target-dedup.go

@@ -0,0 +1,140 @@
+package backend
+
+import (
+	"fmt"
+	"log"
+	"sort"
+	"strings"
+)
+
+type TargetDedupReport struct {
+	Account       string
+	Apply         bool
+	Folders       int
+	Groups        int
+	ExtraCopies   int
+	WithoutID     int
+	Deleted       int
+	FolderReports []TargetDedupFolderReport
+}
+
+type TargetDedupFolderReport struct {
+	Folder      string
+	Groups      int
+	ExtraCopies int
+	WithoutID   int
+	Deletes     []TargetDedupDelete
+}
+
+type TargetDedupDelete struct {
+	UID       uint32
+	MessageID string
+}
+
+func DedupTarget(name string, folders []string, apply bool) (TargetDedupReport, error) {
+	a, err := GetAccount(name)
+	if err != nil {
+		return TargetDedupReport{}, err
+	}
+	dst, err := OpenIMAPTarget(a)
+	if err != nil {
+		return TargetDedupReport{}, err
+	}
+	defer dst.Close()
+	allFolders, err := dst.Folders()
+	if err != nil {
+		return TargetDedupReport{}, err
+	}
+	selected := selectedFolderSet(folders)
+	report := TargetDedupReport{Account: name, Apply: apply}
+	for _, folder := range allFolders {
+		if len(selected) > 0 && !selected[folder.Name] {
+			continue
+		}
+		folderReport, err := dedupTargetFolder(dst, folder.Name, apply)
+		if err != nil {
+			return report, err
+		}
+		report.Folders++
+		report.Groups += folderReport.Groups
+		report.ExtraCopies += folderReport.ExtraCopies
+		report.WithoutID += folderReport.WithoutID
+		report.Deleted += len(folderReport.Deletes)
+		report.FolderReports = append(report.FolderReports, folderReport)
+	}
+	log.Printf("target dedup %s apply=%v folders=%d groups=%d extra=%d deleted=%d without_id=%d",
+		report.Account, report.Apply, report.Folders, report.Groups, report.ExtraCopies, report.Deleted, report.WithoutID)
+	return report, nil
+}
+
+func dedupTargetFolder(dst TargetMailbox, folder string, apply bool) (TargetDedupFolderReport, error) {
+	report := TargetDedupFolderReport{Folder: folder}
+	byID := map[string][]uint32{}
+	for offset := 0; ; offset += mailboxListLimit {
+		headers, err := dst.Headers(folder, mailboxListLimit, offset)
+		if err != nil {
+			return report, fmt.Errorf("%s headers: %w", folder, err)
+		}
+		if len(headers) == 0 {
+			break
+		}
+		for _, header := range headers {
+			id := normalizeMessageID(header.MessageID)
+			if id == "" {
+				report.WithoutID++
+				continue
+			}
+			byID[id] = append(byID[id], header.UID)
+		}
+	}
+	var deleteUIDs []uint32
+	for id, uids := range byID {
+		if len(uids) < 2 {
+			continue
+		}
+		sort.Slice(uids, func(i, j int) bool { return uids[i] < uids[j] })
+		report.Groups++
+		report.ExtraCopies += len(uids) - 1
+		for _, uid := range uids[1:] {
+			report.Deletes = append(report.Deletes, TargetDedupDelete{UID: uid, MessageID: id})
+			deleteUIDs = append(deleteUIDs, uid)
+		}
+	}
+	sort.Slice(report.Deletes, func(i, j int) bool {
+		if report.Deletes[i].MessageID == report.Deletes[j].MessageID {
+			return report.Deletes[i].UID < report.Deletes[j].UID
+		}
+		return strings.Compare(report.Deletes[i].MessageID, report.Deletes[j].MessageID) < 0
+	})
+	if apply && len(deleteUIDs) > 0 {
+		sort.Slice(deleteUIDs, func(i, j int) bool { return deleteUIDs[i] < deleteUIDs[j] })
+		for _, del := range report.Deletes {
+			log.Printf("target dedup delete folder=%s uid=%d message_id=%s", folder, del.UID, del.MessageID)
+		}
+		if err := dst.DeleteUIDs(folder, deleteUIDs); err != nil {
+			return report, fmt.Errorf("%s delete uids: %w", folder, err)
+		}
+	}
+	if report.Groups > 0 || report.WithoutID > 0 {
+		log.Printf("target dedup folder=%s groups=%d extra=%d without_id=%d apply=%v", folder, report.Groups, report.ExtraCopies, report.WithoutID, apply)
+	}
+	return report, nil
+}
+
+func LogTargetDedupReport(report TargetDedupReport) {
+	mode := "DRY-RUN"
+	if report.Apply {
+		mode = "APPLY"
+	}
+	log.Printf("target dedup report %s account=%s folders=%d groups=%d extra=%d deleted=%d without_id=%d",
+		mode, report.Account, report.Folders, report.Groups, report.ExtraCopies, report.Deleted, report.WithoutID)
+	for _, folder := range report.FolderReports {
+		if folder.Groups == 0 && folder.WithoutID == 0 {
+			continue
+		}
+		log.Printf("target dedup report folder=%s groups=%d extra=%d without_id=%d", folder.Folder, folder.Groups, folder.ExtraCopies, folder.WithoutID)
+		for _, del := range folder.Deletes {
+			log.Printf("target dedup report candidate folder=%s uid=%d message_id=%s", folder.Folder, del.UID, del.MessageID)
+		}
+	}
+}