Mail-Graveyard/backend/13-target-dedup.go

package backend

import (
	"fmt"
	"log"
	"sort"
	"strings"
)

type TargetDedupReport struct {
	Account       string
	Apply         bool
	Folders       int
	Groups        int
	ExtraCopies   int
	WithoutID     int
	Deleted       int
	FolderReports []TargetDedupFolderReport
}

type TargetDedupFolderReport struct {
	Folder      string
	Groups      int
	ExtraCopies int
	WithoutID   int
	Deletes     []TargetDedupDelete
}

type TargetDedupDelete struct {
	UID       uint32
	MessageID string
}

func DedupTarget(name string, folders []string, apply bool) (TargetDedupReport, error) {
	a, err := GetAccount(name)
	if err != nil {
		return TargetDedupReport{}, err
	}
	dst, err := OpenIMAPTarget(a)
	if err != nil {
		return TargetDedupReport{}, err
	}
	defer dst.Close()
	allFolders, err := dst.Folders()
	if err != nil {
		return TargetDedupReport{}, err
	}
	selected := selectedFolderSet(folders)
	report := TargetDedupReport{Account: name, Apply: apply}
	for _, folder := range allFolders {
		if len(selected) > 0 && !selected[folder.Name] {
			continue
		}
		folderReport, err := dedupTargetFolder(dst, folder.Name, apply)
		if err != nil {
			return report, err
		}
		report.Folders++
		report.Groups += folderReport.Groups
		report.ExtraCopies += folderReport.ExtraCopies
		report.WithoutID += folderReport.WithoutID
		report.Deleted += len(folderReport.Deletes)
		report.FolderReports = append(report.FolderReports, folderReport)
	}
	log.Printf("target dedup %s apply=%v folders=%d groups=%d extra=%d deleted=%d without_id=%d",
		report.Account, report.Apply, report.Folders, report.Groups, report.ExtraCopies, report.Deleted, report.WithoutID)
	return report, nil
}

func dedupTargetFolder(dst TargetMailbox, folder string, apply bool) (TargetDedupFolderReport, error) {
	report := TargetDedupFolderReport{Folder: folder}
	byID := map[string][]uint32{}
	for offset := 0; ; offset += mailboxListLimit {
		headers, err := dst.Headers(folder, mailboxListLimit, offset)
		if err != nil {
			return report, fmt.Errorf("%s headers: %w", folder, err)
		}
		if len(headers) == 0 {
			break
		}
		for _, header := range headers {
			id := normalizeMessageID(header.MessageID)
			if id == "" {
				report.WithoutID++
				continue
			}
			byID[id] = append(byID[id], header.UID)
		}
	}
	var deleteUIDs []uint32
	for id, uids := range byID {
		if len(uids) < 2 {
			continue
		}
		sort.Slice(uids, func(i, j int) bool { return uids[i] < uids[j] })
		report.Groups++
		report.ExtraCopies += len(uids) - 1
		for _, uid := range uids[1:] {
			report.Deletes = append(report.Deletes, TargetDedupDelete{UID: uid, MessageID: id})
			deleteUIDs = append(deleteUIDs, uid)
		}
	}
	sort.Slice(report.Deletes, func(i, j int) bool {
		if report.Deletes[i].MessageID == report.Deletes[j].MessageID {
			return report.Deletes[i].UID < report.Deletes[j].UID
		}
		return strings.Compare(report.Deletes[i].MessageID, report.Deletes[j].MessageID) < 0
	})
	if apply && len(deleteUIDs) > 0 {
		sort.Slice(deleteUIDs, func(i, j int) bool { return deleteUIDs[i] < deleteUIDs[j] })
		for _, del := range report.Deletes {
			log.Printf("target dedup delete folder=%s uid=%d message_id=%s", folder, del.UID, del.MessageID)
		}
		if err := dst.DeleteUIDs(folder, deleteUIDs); err != nil {
			return report, fmt.Errorf("%s delete uids: %w", folder, err)
		}
	}
	if report.Groups > 0 || report.WithoutID > 0 {
		log.Printf("target dedup folder=%s groups=%d extra=%d without_id=%d apply=%v", folder, report.Groups, report.ExtraCopies, report.WithoutID, apply)
	}
	return report, nil
}

func LogTargetDedupReport(report TargetDedupReport) {
	mode := "DRY-RUN"
	if report.Apply {
		mode = "APPLY"
	}
	log.Printf("target dedup report %s account=%s folders=%d groups=%d extra=%d deleted=%d without_id=%d",
		mode, report.Account, report.Folders, report.Groups, report.ExtraCopies, report.Deleted, report.WithoutID)
	for _, folder := range report.FolderReports {
		if folder.Groups == 0 && folder.WithoutID == 0 {
			continue
		}
		log.Printf("target dedup report folder=%s groups=%d extra=%d without_id=%d", folder.Folder, folder.Groups, folder.ExtraCopies, folder.WithoutID)
		for _, del := range folder.Deletes {
			log.Printf("target dedup report candidate folder=%s uid=%d message_id=%s", folder.Folder, del.UID, del.MessageID)
		}
	}
}