Mail-Graveyard/backend/13-target-dedup.go

package backend

import (
	"fmt"
	"log"
	"sort"
	"strings"
)

type TargetDedupReport struct {
	Account       string
	Apply         bool
	Folders       int
	Groups        int
	ExtraCopies   int
	Candidates    int
	WithoutID     int
	VariantGroups int
	Deleted       int
	FolderReports []TargetDedupFolderReport
}

type TargetDedupFolderReport struct {
	Folder        string
	Groups        int
	ExtraCopies   int
	WithoutID     int
	VariantGroups int
	Deletes       []TargetDedupDelete
}

type TargetDedupDelete struct {
	UID        uint32
	MessageID  string
	BodySHA256 string
}

func DedupTarget(name string, folders []string, apply bool) (TargetDedupReport, error) {
	a, err := GetAccount(name)
	if err != nil {
		return TargetDedupReport{}, err
	}
	dst, err := OpenIMAPTarget(a)
	if err != nil {
		return TargetDedupReport{}, err
	}
	defer dst.Close()
	allFolders, err := dst.Folders()
	if err != nil {
		return TargetDedupReport{}, err
	}
	selected := selectedFolderSet(folders)
	report := TargetDedupReport{Account: name, Apply: apply}
	for _, folder := range allFolders {
		if len(selected) > 0 && !selected[folder.Name] {
			continue
		}
		folderReport, err := dedupTargetFolder(dst, folder.Name, apply)
		if err != nil {
			return report, err
		}
		report.Folders++
		report.Groups += folderReport.Groups
		report.ExtraCopies += folderReport.ExtraCopies
		report.Candidates += len(folderReport.Deletes)
		report.WithoutID += folderReport.WithoutID
		report.VariantGroups += folderReport.VariantGroups
		if apply {
			report.Deleted += len(folderReport.Deletes)
		}
		report.FolderReports = append(report.FolderReports, folderReport)
	}
	log.Printf("target dedup %s apply=%v folders=%d groups=%d extra=%d candidates=%d deleted=%d without_id=%d protected_variant_groups=%d",
		report.Account, report.Apply, report.Folders, report.Groups, report.ExtraCopies, report.Candidates, report.Deleted, report.WithoutID, report.VariantGroups)
	return report, nil
}

func dedupTargetFolder(dst TargetMailbox, folder string, apply bool) (TargetDedupFolderReport, error) {
	report := TargetDedupFolderReport{Folder: folder}
	byID := map[string][]uint32{}
	headers, err := dst.AllHeaders(folder)
	if err != nil {
		return report, fmt.Errorf("%s headers: %w", folder, err)
	}
	for _, header := range headers {
		id := normalizeMessageID(header.MessageID)
		if id == "" {
			report.WithoutID++
			continue
		}
		byID[id] = append(byID[id], header.UID)
	}
	var deleteUIDs []uint32
	for id, uids := range byID {
		if len(uids) < 2 {
			continue
		}
		byBody := map[string][]uint32{}
		for _, uid := range uids {
			msg, err := dst.FetchOne(folder, uid)
			if err != nil {
				return report, fmt.Errorf("%s fetch uid %d for safe dedup: %w", folder, uid, err)
			}
			hash := bodySHA256(msg.Body)
			byBody[hash] = append(byBody[hash], uid)
		}
		if len(byBody) > 1 {
			report.VariantGroups++
		}
		hashes := make([]string, 0, len(byBody))
		for hash := range byBody {
			hashes = append(hashes, hash)
		}
		sort.Strings(hashes)
		for _, hash := range hashes {
			bodyUIDs := byBody[hash]
			if len(bodyUIDs) < 2 {
				continue
			}
			sort.Slice(bodyUIDs, func(i, j int) bool { return bodyUIDs[i] < bodyUIDs[j] })
			report.Groups++
			report.ExtraCopies += len(bodyUIDs) - 1
			for _, uid := range bodyUIDs[1:] {
				report.Deletes = append(report.Deletes, TargetDedupDelete{UID: uid, MessageID: id, BodySHA256: hash})
				deleteUIDs = append(deleteUIDs, uid)
			}
		}
	}
	sort.Slice(report.Deletes, func(i, j int) bool {
		if report.Deletes[i].MessageID == report.Deletes[j].MessageID {
			return report.Deletes[i].UID < report.Deletes[j].UID
		}
		return strings.Compare(report.Deletes[i].MessageID, report.Deletes[j].MessageID) < 0
	})
	if apply && len(deleteUIDs) > 0 {
		sort.Slice(deleteUIDs, func(i, j int) bool { return deleteUIDs[i] < deleteUIDs[j] })
		for _, del := range report.Deletes {
			log.Printf("target dedup delete folder=%s uid=%d message_id=%s body_sha256=%s", folder, del.UID, del.MessageID, del.BodySHA256)
		}
		if err := dst.DeleteUIDs(folder, deleteUIDs); err != nil {
			return report, fmt.Errorf("%s delete uids: %w", folder, err)
		}
	}
	if report.Groups > 0 || report.WithoutID > 0 || report.VariantGroups > 0 {
		log.Printf("target dedup folder=%s groups=%d extra=%d without_id=%d protected_variant_groups=%d apply=%v", folder, report.Groups, report.ExtraCopies, report.WithoutID, report.VariantGroups, apply)
	}
	return report, nil
}

func LogTargetDedupReport(report TargetDedupReport) {
	mode := "DRY-RUN"
	if report.Apply {
		mode = "APPLY"
	}
	log.Printf("target dedup report %s account=%s folders=%d groups=%d extra=%d candidates=%d deleted=%d without_id=%d protected_variant_groups=%d",
		mode, report.Account, report.Folders, report.Groups, report.ExtraCopies, report.Candidates, report.Deleted, report.WithoutID, report.VariantGroups)
	for _, folder := range report.FolderReports {
		if folder.Groups == 0 && folder.WithoutID == 0 && folder.VariantGroups == 0 {
			continue
		}
		log.Printf("target dedup report folder=%s groups=%d extra=%d without_id=%d protected_variant_groups=%d", folder.Folder, folder.Groups, folder.ExtraCopies, folder.WithoutID, folder.VariantGroups)
		for _, del := range folder.Deletes {
			log.Printf("target dedup report candidate folder=%s uid=%d message_id=%s", folder.Folder, del.UID, del.MessageID)
		}
	}
}