Mail-Graveyard/backend/12-archive-tools.go

package backend

import (
	"archive/zip"
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"html"
	"io"
	"net/http"
	"net/mail"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/klauspost/compress/zstd"
)

type ArchiveStats struct {
	Name       string
	Files      int
	Messages   int
	Duplicates int
	Bytes      int64
}

type ArchiveDedupResult struct {
	Removed int
	Files   int
}

func importExportHandler(w http.ResponseWriter, r *http.Request) {
	if !requireManager(w, r) {
		return
	}
	archives, err := ListArchiveMailboxes()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	stats := archiveStatsMap(archives)
	renderImportExportPage(w, r, archives, stats, r.URL.Query().Get("msg"), r.URL.Query().Get("err"))
}

func archiveUploadHandler(w http.ResponseWriter, r *http.Request) {
	if !requireManager(w, r) {
		return
	}
	if r.Method != http.MethodPost {
		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
		return
	}
	archive, err := safeArchiveName(r.FormValue("archive_name"))
	if err != nil {
		redirectImportExport(w, r, err.Error())
		return
	}
	if err := SaveArchiveMailbox(archive); err != nil {
		redirectImportExport(w, r, err.Error())
		return
	}
	dir, err := archiveDir(archive)
	if err != nil {
		redirectImportExport(w, r, err.Error())
		return
	}
	if err := os.MkdirAll(dir, 0o700); err != nil {
		redirectImportExport(w, r, err.Error())
		return
	}
	if err := r.ParseMultipartForm(512 << 20); err != nil {
		redirectImportExport(w, r, err.Error())
		return
	}
	file, header, err := r.FormFile("mailfile")
	if err != nil {
		redirectImportExport(w, r, "Keine Import-Datei gewaehlt.")
		return
	}
	defer file.Close()
	name, err := safeArchiveFileName(header.Filename)
	if err != nil {
		redirectImportExport(w, r, err.Error())
		return
	}
	dst, err := os.OpenFile(filepath.Join(dir, name), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
	if err != nil {
		redirectImportExport(w, r, err.Error())
		return
	}
	defer dst.Close()
	if _, err := io.Copy(dst, file); err != nil {
		redirectImportExport(w, r, err.Error())
		return
	}
	redirectImportExport(w, r, "Import gespeichert.")
}

func archiveDownloadHandler(w http.ResponseWriter, r *http.Request) {
	if !requireManager(w, r) {
		return
	}
	archive, err := safeArchiveName(r.URL.Query().Get("archive_name"))
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	dir, err := archiveDir(archive)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	if _, err := os.Stat(dir); err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	w.Header().Set("Content-Type", "application/zip")
	w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s.zip"`, archive))
	zw := zip.NewWriter(w)
	defer zw.Close()
	_ = filepath.WalkDir(dir, func(path string, entry os.DirEntry, walkErr error) error {
		if walkErr != nil || entry.IsDir() {
			return walkErr
		}
		rel, err := filepath.Rel(dir, path)
		if err != nil {
			return err
		}
		rel = filepath.ToSlash(rel)
		info, err := entry.Info()
		if err != nil {
			return err
		}
		header, err := zip.FileInfoHeader(info)
		if err != nil {
			return err
		}
		header.Name = rel
		if strings.HasSuffix(strings.ToLower(rel), ".mbox.zst") {
			header.Name = strings.TrimSuffix(rel, ".zst")
		}
		header.Method = zip.Deflate
		dst, err := zw.CreateHeader(header)
		if err != nil {
			return err
		}
		src, err := os.Open(path)
		if err != nil {
			return err
		}
		defer src.Close()
		if strings.HasSuffix(strings.ToLower(path), ".mbox.zst") {
			err = exportPlainMbox(src, dst)
		} else {
			_, err = io.Copy(dst, src)
		}
		return err
	})
}

func archiveDedupHandler(w http.ResponseWriter, r *http.Request) {
	if !requireManager(w, r) {
		return
	}
	if r.Method != http.MethodPost {
		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
		return
	}
	archive, err := safeArchiveName(r.FormValue("archive_name"))
	if err != nil {
		redirectArchives(w, r, err.Error())
		return
	}
	result, err := DeduplicateArchive(archive)
	if err != nil {
		redirectArchives(w, r, err.Error())
		return
	}
	redirectArchives(w, r, fmt.Sprintf("Dedup abgeschlossen: %d Dubletten aus %d mbox-Dateien entfernt.", result.Removed, result.Files))
}

func renderImportExportPage(w http.ResponseWriter, r *http.Request, archives []string, stats map[string]ArchiveStats, msg, errMsg string) {
	user := CurrentUser(r)
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	fmt.Fprintf(w, `<!doctype html><html lang="de"><head><meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<title>Import/Export - Mail-Graveyard</title>
<link rel="stylesheet" href="/static/style.css?v=20260712-22">
<script src="/static/htmx.min.js"></script>
<script src="/static/app.js?v=20260712-22" defer></script></head>
<body class="ol2013">
<header class="ribbon">
  <button class="app-menu-button" type="button" aria-label="Mail-Graveyard-Menue" aria-expanded="false" data-backstage-toggle><span class="mail-logo" aria-hidden="true"></span></button>
  <nav class="ribbon-tabs">%s%s</nav>
  %s
</header>
%s
<main class="accounts-page archive-page">`, renderRibbonEmailBoxMenu(""), renderRibbonTransferMenu(""), renderRibbonActions(false, user), renderBackstage("import-export"))
	if msg != "" {
		fmt.Fprintf(w, `<div class="notice ok">%s</div>`, html.EscapeString(msg))
	}
	if errMsg != "" {
		fmt.Fprintf(w, `<div class="notice bad">%s</div>`, html.EscapeString(errMsg))
	}
	fmt.Fprint(w, `<section class="archive-page-layout"><div class="archive-list-panel"><div class="pane-head">Export / Speicher</div>`)
	renderArchiveStatsTable(w, archives, stats)
	fmt.Fprint(w, `</div><div class="archive-form-panel"><div class="pane-head">Import</div>`)
	renderArchiveUploadForm(w, archives)
	fmt.Fprint(w, `</div></section></main><footer class="statusbar"><span>bereit</span></footer></body></html>`)
}

func renderArchiveStatsTable(w http.ResponseWriter, archives []string, stats map[string]ArchiveStats) {
	if len(archives) == 0 {
		fmt.Fprint(w, `<div class="ef-empty">Noch keine Archiv-Mailbox angelegt.</div>`)
		return
	}
	fmt.Fprint(w, `<table class="archive-table"><thead><tr><th>Archiv-Mailbox</th><th>Speicher</th><th>Mails</th><th>Dubletten</th><th></th></tr></thead><tbody>`)
	for _, archive := range archives {
		stat := stats[archive]
		fmt.Fprintf(w, `<tr><td>%s</td><td>%s</td><td>%d</td><td>%d</td><td class="archive-action"><a class="btn compact secondary" href="/import-export/download?archive_name=%s">Download</a></td></tr>`,
			html.EscapeString(archive), html.EscapeString(formatBytes(stat.Bytes)), stat.Messages, stat.Duplicates, urlQuery(archive))
	}
	fmt.Fprint(w, `</tbody></table>`)
}

func renderArchiveUploadForm(w http.ResponseWriter, archives []string) {
	fmt.Fprint(w, `<form class="archive-create-panel" method="post" action="/import-export/upload" enctype="multipart/form-data">
<label class="label">Archiv-Mailbox`)
	fmt.Fprint(w, archiveUploadSelect(archives))
	fmt.Fprint(w, `</label>
<label class="label">Datei<input class="input" type="file" name="mailfile" accept=".mbox,.pst,.zst" required></label>
<button class="btn" type="submit">Upload</button>
</form>
<div class="archive-help">
  <div class="archive-help-title">Import/Export</div>
  <p>mbox-Dateien werden direkt als Ordnerinhalt importiert und sind im Viewer nutzbar.</p>
  <p>PST-Dateien werden als Archivdatei abgelegt; eine PST-zu-mbox-Konvertierung braucht spaeter einen eigenen Parser.</p>
  <p>Download exportiert die komplette Archiv-Mailbox als ZIP.</p>
</div>`)
}

func archiveUploadSelect(archives []string) string {
	if len(archives) == 0 {
		return `<select name="archive_name" disabled><option>erst Archiv-Mailbox erstellen</option></select>`
	}
	var b strings.Builder
	b.WriteString(`<select name="archive_name">`)
	for _, archive := range archives {
		fmt.Fprintf(&b, `<option value="%s">%s</option>`, html.EscapeString(archive), html.EscapeString(archive))
	}
	b.WriteString(`</select>`)
	return b.String()
}

func archiveStatsMap(archives []string) map[string]ArchiveStats {
	out := map[string]ArchiveStats{}
	for _, archive := range archives {
		stat, err := ArchiveStatsFor(archive)
		if err == nil {
			out[archive] = stat
		} else {
			out[archive] = ArchiveStats{Name: archive}
		}
	}
	return out
}

func ArchiveStatsFor(name string) (ArchiveStats, error) {
	dir, err := archiveDir(name)
	if err != nil {
		return ArchiveStats{}, err
	}
	stat := ArchiveStats{Name: name}
	seen := map[string]int{}
	if err := filepath.WalkDir(dir, func(path string, entry os.DirEntry, walkErr error) error {
		if walkErr != nil {
			return walkErr
		}
		if entry.IsDir() {
			return nil
		}
		info, err := entry.Info()
		if err != nil {
			return err
		}
		stat.Files++
		stat.Bytes += info.Size()
		if isMboxArchivePath(path) {
			msgs, err := readMboxMessages(path)
			if err != nil {
				return nil
			}
			stat.Messages += len(msgs)
			for _, msg := range msgs {
				key := messageDedupKey(msg)
				seen[key]++
				if seen[key] > 1 {
					stat.Duplicates++
				}
			}
		}
		return nil
	}); err != nil && !os.IsNotExist(err) {
		return stat, err
	}
	return stat, nil
}

func DeduplicateArchive(name string) (ArchiveDedupResult, error) {
	dir, err := archiveDir(name)
	if err != nil {
		return ArchiveDedupResult{}, err
	}
	seen := map[string]bool{}
	var result ArchiveDedupResult
	var mboxes []string
	if err := filepath.WalkDir(dir, func(path string, entry os.DirEntry, walkErr error) error {
		if walkErr != nil {
			return walkErr
		}
		if !entry.IsDir() && strings.EqualFold(filepath.Ext(path), ".mbox") {
			mboxes = append(mboxes, path)
		}
		return nil
	}); err != nil {
		return result, err
	}
	sortStrings(mboxes)
	for _, path := range mboxes {
		msgs, err := readMboxMessages(path)
		if err != nil {
			return result, err
		}
		kept := make([][]byte, 0, len(msgs))
		for _, msg := range msgs {
			key := messageDedupKey(msg)
			if seen[key] {
				result.Removed++
				continue
			}
			seen[key] = true
			kept = append(kept, msg)
		}
		if len(kept) != len(msgs) {
			if err := writeMboxMessages(path, kept); err != nil {
				return result, err
			}
		}
		result.Files++
	}
	return result, nil
}

func messageDedupKey(raw []byte) string {
	msg, err := mail.ReadMessage(bytes.NewReader(raw))
	if err == nil {
		id := strings.ToLower(strings.TrimSpace(msg.Header.Get("Message-ID")))
		if id != "" {
			return "id:" + id
		}
	}
	sum := sha256.Sum256(bytes.TrimSpace(raw))
	return "sha256:" + hex.EncodeToString(sum[:])
}

func writeMboxMessages(path string, messages [][]byte) error {
	tmp := path + ".tmp"
	f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
	if err != nil {
		return err
	}
	for _, raw := range messages {
		if _, err := fmt.Fprintf(f, "From MAILER-DAEMON %s\r\n", time.Now().Format("Mon Jan _2 15:04:05 2006")); err != nil {
			_ = f.Close()
			return err
		}
		body := bytes.ReplaceAll(raw, []byte("\r\nFrom "), []byte("\r\n>From "))
		body = bytes.ReplaceAll(body, []byte("\nFrom "), []byte("\n>From "))
		if _, err := f.Write(body); err != nil {
			_ = f.Close()
			return err
		}
		if !bytes.HasSuffix(body, []byte("\n")) {
			if _, err := f.WriteString("\r\n"); err != nil {
				_ = f.Close()
				return err
			}
		}
		if _, err := f.WriteString("\r\n"); err != nil {
			_ = f.Close()
			return err
		}
	}
	if err := f.Close(); err != nil {
		return err
	}
	return os.Rename(tmp, path)
}

func archiveDir(name string) (string, error) {
	name, err := safeArchiveName(name)
	if err != nil {
		return "", err
	}
	root, err := filepath.Abs(Cfg.MboxRoot)
	if err != nil {
		return "", err
	}
	path, err := filepath.Abs(filepath.Join(root, name))
	if err != nil {
		return "", err
	}
	if path != root && !strings.HasPrefix(path, root+string(os.PathSeparator)) {
		return "", fmt.Errorf("ungueltiger Archiv-Pfad")
	}
	return path, nil
}

func safeArchiveFileName(value string) (string, error) {
	name := filepath.Base(strings.TrimSpace(value))
	if name == "" || name == "." || name == ".." {
		return "", fmt.Errorf("Dateiname fehlt.")
	}
	lower := strings.ToLower(name)
	ext := strings.ToLower(filepath.Ext(name))
	if ext != ".mbox" && ext != ".pst" && !strings.HasSuffix(lower, ".mbox.zst") {
		return "", fmt.Errorf("Nur mbox-, mbox.zst- und PST-Dateien koennen importiert werden.")
	}
	clean := strings.NewReplacer("\\", "_", "/", "_", ":", "_").Replace(name)
	if strings.HasSuffix(strings.ToLower(clean), ".mbox.zst") {
		if strings.TrimSuffix(strings.TrimSuffix(clean, ".zst"), ".mbox") == "" {
			clean = "import.mbox.zst"
		}
	} else if strings.TrimSuffix(clean, ext) == "" {
		clean = "import" + ext
	}
	return clean, nil
}

func isMboxArchivePath(path string) bool {
	lower := strings.ToLower(path)
	return strings.HasSuffix(lower, ".mbox") || strings.HasSuffix(lower, ".mbox.zst")
}

func exportPlainMbox(src io.Reader, dst io.Writer) error {
	dec, err := zstd.NewReader(src)
	if err != nil {
		return err
	}
	defer dec.Close()
	_, err = io.Copy(dst, dec)
	return err
}

func formatBytes(n int64) string {
	const unit = 1024
	if n < unit {
		return fmt.Sprintf("%d B", n)
	}
	div, exp := int64(unit), 0
	for n >= div*unit && exp < 4 {
		div *= unit
		exp++
	}
	return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
}

func redirectImportExport(w http.ResponseWriter, r *http.Request, msg string) {
	key := "msg"
	if strings.Contains(strings.ToLower(msg), "fehl") || strings.Contains(strings.ToLower(msg), "ungueltig") || strings.Contains(strings.ToLower(msg), "error") {
		key = "err"
	}
	http.Redirect(w, r, "/import-export?"+key+"="+urlQuery(msg), http.StatusSeeOther)
}