Mail-Graveyard/backend/06-mbox.go

package backend

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"html"
	"io"
	"mime"
	"mime/multipart"
	"mime/quotedprintable"
	"net/mail"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/klauspost/compress/zstd"
)

// mbox ist das lokale Cold-Backup-Format: eine Datei pro IMAP-Ordner, der
// Ordnerbaum 1:1 auf der Platte. eM Client / Thunderbird importieren mbox
// direkt. (Kein PST -- das laesst sich in Go nicht lean schreiben.)

// MboxWriter haengt Nachrichten an die mbox-Datei eines Ordners an.
type MboxWriter struct {
	root string
}

type MboxAppendInfo struct {
	Path        string
	FileOffset  int64
	FrameLen    int64
	InnerOffset int64
	InnerLen    int64
	Subject     string
	From        string
	Date        string
}

// NewMboxWriter legt <mboxRoot>/<account>/ an und spiegelt den Ordnerbaum.
func NewMboxWriter(dir string) (*MboxWriter, error) {
	if err := os.MkdirAll(dir, 0o700); err != nil {
		return nil, err
	}
	return &MboxWriter{root: dir}, nil
}

// Append schreibt eine Mail im mbox-Format ("From "-Trennzeile, >From-Quoting)
// in <ordner>.mbox.
func (w *MboxWriter) Append(folder string, m RawMessage) (MboxAppendInfo, error) {
	record := mboxRecord(m)
	info := mboxAppendInfoFromMessage(m, record)
	path := filepath.Join(w.root, safeMboxName(folder)+mboxFileExt())
	info.Path = path
	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
		return info, err
	}
	f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600)
	if err != nil {
		return info, err
	}
	defer f.Close()
	offset, err := f.Seek(0, io.SeekEnd)
	if err != nil {
		return info, err
	}
	info.FileOffset = offset
	var written int
	if Cfg.MboxCompression == "zstd" {
		enc, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedDefault))
		if err != nil {
			return info, err
		}
		compressed := enc.EncodeAll(record, nil)
		written, err = f.Write(compressed)
		info.FrameLen = int64(len(compressed))
	} else {
		written, err = f.Write(record)
		info.FrameLen = int64(written)
	}
	if err != nil {
		return info, err
	}
	if written != int(info.FrameLen) {
		return info, io.ErrShortWrite
	}
	return info, nil
}

func mboxFileExt() string {
	if Cfg.MboxCompression == "zstd" {
		return ".mbox.zst"
	}
	return ".mbox"
}

func mboxRecord(m RawMessage) []byte {
	var b bytes.Buffer
	fromDate := m.InternalDate
	if fromDate.IsZero() {
		fromDate = time.Now()
	}
	fmt.Fprintf(&b, "From MAILER-DAEMON %s\r\n", fromDate.Format("Mon Jan _2 15:04:05 2006"))
	body := bytes.ReplaceAll(m.Body, []byte("\r\nFrom "), []byte("\r\n>From "))
	body = bytes.ReplaceAll(body, []byte("\nFrom "), []byte("\n>From "))
	_, _ = b.Write(body)
	if !bytes.HasSuffix(body, []byte("\n")) {
		_, _ = b.WriteString("\r\n")
	}
	_, _ = b.WriteString("\r\n")
	return b.Bytes()
}

func mboxAppendInfoFromMessage(m RawMessage, record []byte) MboxAppendInfo {
	info := MboxAppendInfo{InnerLen: int64(len(record))}
	parts := splitMboxRecordsWithOffsets(record)
	if len(parts) > 0 {
		info.InnerOffset = parts[0].Offset
		info.InnerLen = int64(len(parts[0].Message))
	}
	msg, err := mail.ReadMessage(bytes.NewReader(m.Body))
	if err != nil {
		return info
	}
	info.Subject = decodeHeader(msg.Header.Get("Subject"))
	info.From = decodeHeader(msg.Header.Get("From"))
	info.Date = decodeHeader(msg.Header.Get("Date"))
	return info
}

// --- Viewer-Seite: mbox wieder lesen fuers Browser-Betrachten (08-viewer.go) ---

// MboxEntry ist eine Nachricht wie im Listen-Pane angezeigt.
type MboxEntry struct {
	Index   int
	From    string
	Subject string
	Date    string
}

// ReadMboxList parst die Kopfzeilen aller Mails einer mbox-Datei (fuer die
// Nachrichtenliste). ReadMboxMessage liefert eine einzelne Mail als Rohtext.
func ReadMboxList(path string) ([]MboxEntry, error) {
	if err := EnsureMboxIndexed(path); err != nil {
		return nil, err
	}
	if entries, ok := readMboxListFromIndex(path); ok {
		return entries, nil
	}
	msgs, err := readMboxMessages(path)
	if err != nil {
		return nil, err
	}
	out := make([]MboxEntry, 0, len(msgs))
	for i, raw := range msgs {
		msg, err := mail.ReadMessage(bytes.NewReader(raw))
		if err != nil {
			out = append(out, MboxEntry{Index: i, From: "(unlesbar)", Subject: "Unlesbare Nachricht"})
			continue
		}
		out = append(out, MboxEntry{
			Index:   i,
			From:    decodeHeader(msg.Header.Get("From")),
			Subject: decodeHeader(msg.Header.Get("Subject")),
			Date:    decodeHeader(msg.Header.Get("Date")),
		})
	}
	return out, nil
}

func ReadMboxMessage(path string, index int) ([]byte, error) {
	if err := EnsureMboxIndexed(path); err != nil {
		return nil, err
	}
	if raw, ok, err := readMboxMessageFromIndex(path, index); ok || err != nil {
		return raw, err
	}
	msgs, err := readMboxMessages(path)
	if err != nil {
		return nil, err
	}
	if index < 0 || index >= len(msgs) {
		return nil, fmt.Errorf("message index out of range")
	}
	return msgs[index], nil
}

func readMboxMessages(path string) ([][]byte, error) {
	b, err := os.ReadFile(path)
	if err != nil {
		return nil, err
	}
	if isZstdMbox(path, b) {
		dec, err := zstd.NewReader(nil)
		if err != nil {
			return nil, err
		}
		defer dec.Close()
		b, err = dec.DecodeAll(b, nil)
		if err != nil {
			return nil, err
		}
	}
	return readMboxMessagesBytes(b), nil
}

func EnsureMboxIndexed(path string) error {
	accountID, folder, ok := mboxIndexContext(path)
	if !ok {
		return nil
	}
	info, err := os.Stat(path)
	if err != nil {
		return err
	}
	indexed, err := MboxIndexedBytes(accountID, folder)
	if err != nil {
		return err
	}
	if indexed == info.Size() {
		return nil
	}
	return ReindexMboxFile(accountID, folder, path)
}

func ReindexMboxFile(accountID int64, folder, path string) error {
	info, err := os.Stat(path)
	if err != nil {
		return err
	}
	var entries []MboxIndexEntry
	if strings.HasSuffix(strings.ToLower(path), ".zst") {
		entries, err = reindexZstdMbox(path)
	} else {
		entries, err = reindexPlainMbox(path)
	}
	if err != nil {
		return err
	}
	return ReplaceMboxIndex(accountID, folder, entries, info.Size())
}

func ReindexArchives(name string) error {
	name = strings.TrimSpace(name)
	if name == "" {
		return fmt.Errorf("reindex ziel fehlt")
	}
	accounts, err := ListAccounts()
	if err != nil {
		return err
	}
	matched := 0
	for _, account := range accounts {
		archive := accountMboxDir(account)
		if archive == "" {
			continue
		}
		if name != "all" && !strings.EqualFold(name, account.Name) && !strings.EqualFold(name, archive) {
			continue
		}
		if err := reindexAccountArchive(account); err != nil {
			return err
		}
		matched++
	}
	if matched == 0 {
		return fmt.Errorf("kein Konto/Archiv fuer reindex %q gefunden", name)
	}
	return nil
}

func reindexAccountArchive(account Account) error {
	dir := filepath.Join(Cfg.MboxRoot, accountMboxDir(account))
	for _, path := range archiveMboxPaths(dir) {
		folder := folderNameFromMboxPath(path)
		if folder == "" {
			continue
		}
		if err := ReindexMboxFile(account.ID, folder, path); err != nil {
			return fmt.Errorf("%s %s: %w", account.Name, folder, err)
		}
	}
	return nil
}

func reindexPlainMbox(path string) ([]MboxIndexEntry, error) {
	b, err := os.ReadFile(path)
	if err != nil {
		return nil, err
	}
	var out []MboxIndexEntry
	for _, part := range splitMboxRecordsWithOffsets(b) {
		entry := indexEntryFromMboxRecord(part.Message, part.Offset, int64(part.Length), 0, int64(part.Length))
		if entry.MessageID != "" {
			out = append(out, entry)
		}
	}
	return out, nil
}

func reindexZstdMbox(path string) ([]MboxIndexEntry, error) {
	b, err := os.ReadFile(path)
	if err != nil {
		return nil, err
	}
	dec, err := zstd.NewReader(nil)
	if err != nil {
		return nil, err
	}
	defer dec.Close()
	var out []MboxIndexEntry
	for _, frame := range splitZstdFrames(b) {
		record, err := dec.DecodeAll(frame.Data, nil)
		if err != nil {
			return nil, err
		}
		for _, part := range splitMboxRecordsWithOffsets(record) {
			entry := indexEntryFromMboxRecord(part.Message, frame.Offset, int64(frame.Length), part.Offset, int64(len(part.Message)))
			if entry.MessageID != "" {
				out = append(out, entry)
			}
		}
	}
	return out, nil
}

type mboxRecordPart struct {
	Offset  int64
	Length  int
	Message []byte
}

func splitMboxRecordsWithOffsets(b []byte) []mboxRecordPart {
	normalized := bytes.ReplaceAll(b, []byte("\r\n"), []byte("\n"))
	lines := bytes.SplitAfter(normalized, []byte("\n"))
	var out []mboxRecordPart
	var cur bytes.Buffer
	var curStart int64
	var pos int64
	inMsg := false
	for _, lineWithNL := range lines {
		line := bytes.TrimSuffix(lineWithNL, []byte("\n"))
		if bytes.HasPrefix(line, []byte("From ")) {
			if inMsg && cur.Len() > 0 {
				msg := bytes.TrimRight(cur.Bytes(), "\n")
				out = append(out, mboxRecordPart{Offset: curStart, Length: int(pos - curStart), Message: append([]byte(nil), msg...)})
				cur.Reset()
			}
			inMsg = true
			pos += int64(len(lineWithNL))
			curStart = pos
			continue
		}
		if inMsg {
			if bytes.HasPrefix(line, []byte(">From ")) {
				line = line[1:]
			}
			_, _ = cur.Write(line)
			_ = cur.WriteByte('\n')
		}
		pos += int64(len(lineWithNL))
	}
	if inMsg && cur.Len() > 0 {
		msg := bytes.TrimRight(cur.Bytes(), "\n")
		out = append(out, mboxRecordPart{Offset: curStart, Length: int(pos - curStart), Message: append([]byte(nil), msg...)})
	}
	return out
}

type zstdFramePart struct {
	Offset int64
	Length int
	Data   []byte
}

func splitZstdFrames(b []byte) []zstdFramePart {
	magic := []byte{0x28, 0xb5, 0x2f, 0xfd}
	var starts []int
	for i := 0; i <= len(b)-len(magic); i++ {
		if bytes.Equal(b[i:i+len(magic)], magic) {
			starts = append(starts, i)
		}
	}
	if len(starts) == 0 {
		return nil
	}
	out := make([]zstdFramePart, 0, len(starts))
	for i, start := range starts {
		end := len(b)
		if i+1 < len(starts) {
			end = starts[i+1]
		}
		out = append(out, zstdFramePart{Offset: int64(start), Length: end - start, Data: b[start:end]})
	}
	return out
}

func indexEntryFromMboxRecord(raw []byte, fileOffset, frameLen, innerOffset, innerLen int64) MboxIndexEntry {
	entry := MboxIndexEntry{FileOffset: fileOffset, FrameLen: frameLen, InnerOffset: innerOffset, InnerLen: innerLen}
	msg, err := mail.ReadMessage(bytes.NewReader(raw))
	if err != nil {
		return entry
	}
	entry.MessageID = normalizeMessageID(msg.Header.Get("Message-ID"))
	if entry.MessageID == "" {
		entry.MessageID = messageID(raw)
	}
	entry.Subject = decodeHeader(msg.Header.Get("Subject"))
	entry.From = decodeHeader(msg.Header.Get("From"))
	entry.Date = decodeHeader(msg.Header.Get("Date"))
	return entry
}

func unescapeMboxMessage(raw []byte) []byte {
	raw = bytes.ReplaceAll(raw, []byte("\r\n"), []byte("\n"))
	lines := bytes.Split(raw, []byte("\n"))
	var b bytes.Buffer
	for i, line := range lines {
		if i > 0 {
			_ = b.WriteByte('\n')
		}
		if bytes.HasPrefix(line, []byte(">From ")) {
			line = line[1:]
		}
		_, _ = b.Write(line)
	}
	return bytes.TrimRight(b.Bytes(), "\n")
}

func readMboxMessagesBytes(b []byte) [][]byte {
	b = bytes.ReplaceAll(b, []byte("\r\n"), []byte("\n"))
	lines := bytes.Split(b, []byte("\n"))
	var msgs [][]byte
	var cur bytes.Buffer
	inMsg := false
	for _, line := range lines {
		if bytes.HasPrefix(line, []byte("From ")) {
			if inMsg && cur.Len() > 0 {
				msgs = append(msgs, bytes.TrimRight(cur.Bytes(), "\n"))
				cur.Reset()
			}
			inMsg = true
			continue
		}
		if !inMsg {
			continue
		}
		if bytes.HasPrefix(line, []byte(">From ")) {
			line = line[1:]
		}
		_, _ = cur.Write(line)
		_ = cur.WriteByte('\n')
	}
	if inMsg && cur.Len() > 0 {
		msgs = append(msgs, bytes.TrimRight(cur.Bytes(), "\n"))
	}
	return msgs
}

func readMboxListFromIndex(path string) ([]MboxEntry, bool) {
	accountID, folder, ok := mboxIndexContext(path)
	if !ok {
		return nil, false
	}
	index, err := ListMboxIndex(accountID, folder)
	if err != nil || len(index) == 0 {
		return nil, false
	}
	out := make([]MboxEntry, 0, len(index))
	for _, entry := range index {
		out = append(out, MboxEntry{
			Index:   entry.Seq,
			From:    entry.From,
			Subject: entry.Subject,
			Date:    entry.Date,
		})
	}
	return out, true
}

func readMboxMessageFromIndex(path string, index int) ([]byte, bool, error) {
	accountID, folder, ok := mboxIndexContext(path)
	if !ok {
		return nil, false, nil
	}
	entry, err := GetMboxIndex(accountID, folder, index)
	if err != nil {
		return nil, false, nil
	}
	if !strings.HasSuffix(strings.ToLower(path), ".zst") {
		return nil, false, nil
	}
	f, err := os.Open(path)
	if err != nil {
		return nil, true, err
	}
	defer f.Close()
	if _, err := f.Seek(entry.FileOffset, io.SeekStart); err != nil {
		return nil, true, err
	}
	frame := make([]byte, entry.FrameLen)
	if _, err := io.ReadFull(f, frame); err != nil {
		return nil, true, err
	}
	dec, err := zstd.NewReader(nil)
	if err != nil {
		return nil, true, err
	}
	defer dec.Close()
	record, err := dec.DecodeAll(frame, nil)
	if err != nil {
		return nil, true, err
	}
	start := entry.InnerOffset
	end := entry.InnerOffset + entry.InnerLen
	if start < 0 || end > int64(len(record)) || start >= end {
		return nil, true, fmt.Errorf("message index out of range")
	}
	return unescapeMboxMessage(record[start:end]), true, nil
}

func mboxIndexContext(path string) (int64, string, bool) {
	if DB == nil || Cfg.MboxRoot == "" {
		return 0, "", false
	}
	root, err := filepath.Abs(Cfg.MboxRoot)
	if err != nil {
		return 0, "", false
	}
	abs, err := filepath.Abs(path)
	if err != nil {
		return 0, "", false
	}
	rel, err := filepath.Rel(root, abs)
	if err != nil || strings.HasPrefix(rel, "..") {
		return 0, "", false
	}
	parts := strings.Split(rel, string(os.PathSeparator))
	if len(parts) < 2 {
		return 0, "", false
	}
	account, ok := accountByMboxDir(parts[0])
	if !ok || account.ID == 0 {
		return 0, "", false
	}
	folder := folderNameFromMboxPath(path)
	if folder == "" {
		return 0, "", false
	}
	return account.ID, folder, true
}

func isZstdMbox(path string, b []byte) bool {
	if strings.HasSuffix(strings.ToLower(path), ".zst") {
		return true
	}
	return len(b) >= 4 && b[0] == 0x28 && b[1] == 0xb5 && b[2] == 0x2f && b[3] == 0xfd
}

func decodeHeader(v string) string {
	v = strings.TrimSpace(v)
	if v == "" {
		return ""
	}
	decoded, err := new(mime.WordDecoder).DecodeHeader(v)
	if err != nil {
		return v
	}
	return decoded
}

func messageBody(raw []byte) string {
	msg, err := mail.ReadMessage(bytes.NewReader(raw))
	if err != nil {
		return string(raw)
	}
	body, err := io.ReadAll(msg.Body)
	if err != nil {
		return string(raw)
	}
	return extractTextBody(msg.Header, body)
}

type headerGetter interface {
	Get(string) string
}

func extractTextBody(h headerGetter, body []byte) string {
	mediaType, params, _ := mime.ParseMediaType(h.Get("Content-Type"))
	decoded := decodeTransfer(body, h.Get("Content-Transfer-Encoding"))
	if strings.HasPrefix(mediaType, "multipart/") {
		boundary := params["boundary"]
		if boundary == "" {
			return string(decoded)
		}
		mr := multipart.NewReader(bytes.NewReader(decoded), boundary)
		var htmlBody string
		for {
			part, err := mr.NextPart()
			if err == io.EOF {
				break
			}
			if err != nil {
				break
			}
			partBody, _ := io.ReadAll(part)
			text := extractTextBody(part.Header, partBody)
			partType, _, _ := mime.ParseMediaType(part.Header.Get("Content-Type"))
			if strings.HasPrefix(partType, "text/plain") && strings.TrimSpace(text) != "" {
				return text
			}
			if strings.HasPrefix(partType, "text/html") && htmlBody == "" {
				htmlBody = text
			}
		}
		if htmlBody != "" {
			return htmlBody
		}
		return ""
	}
	if strings.HasPrefix(mediaType, "text/html") || looksLikeHTML(decoded) {
		return htmlToText(string(decoded))
	}
	return string(decoded)
}

func looksLikeHTML(body []byte) bool {
	s := strings.ToLower(strings.TrimSpace(string(body)))
	return strings.HasPrefix(s, "<!doctype html") || strings.HasPrefix(s, "<html") || strings.Contains(s, "<body")
}

func htmlToText(s string) string {
	s = stripHTMLBlock(s, "script")
	s = stripHTMLBlock(s, "style")
	s = markHTMLBreaks(s)
	var b strings.Builder
	inTag := false
	lastSpace := false
	for _, r := range s {
		switch r {
		case '<':
			inTag = true
			if !lastSpace {
				b.WriteByte(' ')
				lastSpace = true
			}
		case '>':
			inTag = false
		default:
			if inTag {
				continue
			}
			if r == '\n' {
				b.WriteByte('\n')
				lastSpace = true
				continue
			}
			if r == '\r' || r == '\t' || r == ' ' {
				if !lastSpace {
					b.WriteByte(' ')
					lastSpace = true
				}
				continue
			}
			b.WriteRune(r)
			lastSpace = false
		}
	}
	text := html.UnescapeString(b.String())
	lines := strings.Split(text, "\n")
	out := make([]string, 0, len(lines))
	for _, line := range lines {
		line = strings.TrimSpace(line)
		if line != "" {
			out = append(out, line)
		}
	}
	if len(out) == 0 {
		return strings.TrimSpace(text)
	}
	return strings.Join(out, "\n")
}

func markHTMLBreaks(s string) string {
	repls := []struct {
		Needle string
		With   string
	}{
		{"<br>", "\n"},
		{"<br/>", "\n"},
		{"<br />", "\n"},
		{"</p>", "\n"},
		{"</div>", "\n"},
		{"</tr>", "\n"},
		{"</td>", " "},
		{"</th>", " "},
		{"</li>", "\n"},
	}
	for _, repl := range repls {
		s = replaceCaseInsensitive(s, repl.Needle, repl.With)
	}
	return s
}

func replaceCaseInsensitive(s, old, new string) string {
	var b strings.Builder
	lower := strings.ToLower(s)
	needle := strings.ToLower(old)
	for {
		i := strings.Index(lower, needle)
		if i < 0 {
			b.WriteString(s)
			return b.String()
		}
		b.WriteString(s[:i])
		b.WriteString(new)
		cut := i + len(old)
		s = s[cut:]
		lower = lower[cut:]
	}
}

func stripHTMLBlock(s, tag string) string {
	lower := strings.ToLower(s)
	open := "<" + tag
	close := "</" + tag + ">"
	for {
		start := strings.Index(lower, open)
		if start < 0 {
			return s
		}
		end := strings.Index(lower[start:], close)
		if end < 0 {
			return s[:start]
		}
		end += start + len(close)
		s = s[:start] + " " + s[end:]
		lower = strings.ToLower(s)
	}
}

func decodeTransfer(body []byte, enc string) []byte {
	switch strings.ToLower(strings.TrimSpace(enc)) {
	case "quoted-printable":
		b, err := io.ReadAll(quotedprintable.NewReader(bytes.NewReader(body)))
		if err == nil {
			return b
		}
	case "base64":
		cleaned := strings.NewReplacer("\r", "", "\n", "", " ", "", "\t", "").Replace(string(body))
		b, err := base64.StdEncoding.DecodeString(cleaned)
		if err == nil {
			return b
		}
	}
	return body
}

func safeMboxName(folder string) string {
	folder = strings.TrimSpace(folder)
	folder = strings.ReplaceAll(folder, "\\", "_")
	folder = strings.ReplaceAll(folder, "/", "_")
	folder = strings.ReplaceAll(folder, ":", "_")
	if folder == "" {
		return "INBOX"
	}
	return folder
}

func archiveMboxPaths(dir string) []string {
	plain, _ := filepath.Glob(filepath.Join(dir, "*.mbox"))
	compressed, _ := filepath.Glob(filepath.Join(dir, "*.mbox.zst"))
	seen := map[string]bool{}
	out := make([]string, 0, len(plain)+len(compressed))
	for _, path := range plain {
		seen[folderNameFromMboxPath(path)] = true
		out = append(out, path)
	}
	for _, path := range compressed {
		if seen[folderNameFromMboxPath(path)] {
			continue
		}
		out = append(out, path)
	}
	sortStrings(out)
	return out
}

func folderNameFromMboxPath(path string) string {
	name := filepath.Base(path)
	name = strings.TrimSuffix(name, ".zst")
	name = strings.TrimSuffix(name, ".mbox")
	return name
}